enso-org/enso
1
1
Fork 0
mirror of https://github.com/enso-org/enso.git synced 2024-11-22 11:52:59 +03:00
Code Issues Projects Releases Wiki Activity

Syntactic synchronization, automatic parentheses, metadata in Ast (#8893)

Browse Source 
- Synchronize Y.Js clients by AST (implements #8237).
- Before committing an edit, insert any parentheses-nodes needed for the concrete syntax to reflect tree structure (fixes #8884).
- Move `externalId` and all node metadata into a Y.Map owned by each `Ast`. This allows including metadata changes in an edit, enables Y.Js merging of changes to different metadata fields, and will enable the use of Y.Js objects in metadata. (Implements #8804.)

### Important Notes

- Metadata is now set and retrieved through accessors on the `Ast` objects.
- Since some metadata edits need to take effect in real time (e.g. node dragging), new lower-overhead APIs (`commitDirect`, `skipTreeRepair`) are provided for careful use in certain cases.
- The client is now bundled as ESM.
- The build script cleans up git-untracked generated files in an outdated location, which fixes lint errors related to `src/generated` that may occur when switching branches.
This commit is contained in:
Kaz Wesley 2024-02-02 01:22:18 -08:00 committed by GitHub
parent 340a3eec4e
commit 343a644051
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
59 changed files with 4471 additions and 3546 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.prettierignore
View File

@ -38,6 +38,7 @@ app/ide-desktop/lib/dashboard/playwright/.cache/
app/ide-desktop/lib/dashboard/dist/
app/gui/view/documentation/assets/stylesheet.css
app/gui2/rust-ffi/pkg
app/gui2/rust-ffi/node-pkg
app/gui2/src/assets/font-*.css
Cargo.lock
build.json

45
app/gui2/e2e/edgeRendering.spec.ts
View File

@ -1,4 +1,4 @@
import { expect, Page, test } from '@playwright/test'
import { expect, test, type Page } from '@playwright/test'
import * as actions from './actions'
import * as locate from './locate'
@ -13,28 +13,29 @@ async function edgesToNodeWithBinding(page: Page, binding: string) {
return page.locator(`[data-target-node-id="${nodeId}"]`)
}
// For each outgoing edge we expect two elements: an element for io and an element for the rendered edge itself.
const EDGE_PARTS = 2
test('Existence of edges between nodes', async ({ page }) => {
await actions.goToGraph(page)
// For each outgoing edge we expect three elements: an element for io, an element for the arrow, and an element for the rendered edge itself.
await expect(await edgesFromNodeWithBinding(page, 'aggregated')).toHaveCount(0)
await expect(await edgesFromNodeWithBinding(page, 'filtered')).toHaveCount(0)
await expect(await edgesFromNodeWithBinding(page, 'data')).toHaveCount(6)
await expect(await edgesFromNodeWithBinding(page, 'data')).toHaveCount(2 * EDGE_PARTS)
await expect(await edgesFromNodeWithBinding(page, 'list')).toHaveCount(0)
await expect(await edgesFromNodeWithBinding(page, 'final')).toHaveCount(0)
await expect(await edgesFromNodeWithBinding(page, 'prod')).toHaveCount(3)
await expect(await edgesFromNodeWithBinding(page, 'sum')).toHaveCount(3)
await expect(await edgesFromNodeWithBinding(page, 'ten')).toHaveCount(3)
await expect(await edgesFromNodeWithBinding(page, 'five')).toHaveCount(3)
await expect(await edgesFromNodeWithBinding(page, 'prod')).toHaveCount(EDGE_PARTS)
await expect(await edgesFromNodeWithBinding(page, 'sum')).toHaveCount(EDGE_PARTS)
await expect(await edgesFromNodeWithBinding(page, 'ten')).toHaveCount(EDGE_PARTS)
await expect(await edgesFromNodeWithBinding(page, 'five')).toHaveCount(EDGE_PARTS)
await expect(await edgesToNodeWithBinding(page, 'aggregated')).toHaveCount(3)
await expect(await edgesToNodeWithBinding(page, 'filtered')).toHaveCount(3)
await expect(await edgesToNodeWithBinding(page, 'aggregated')).toHaveCount(EDGE_PARTS)
await expect(await edgesToNodeWithBinding(page, 'filtered')).toHaveCount(EDGE_PARTS)
await expect(await edgesToNodeWithBinding(page, 'data')).toHaveCount(0)
await expect(await edgesToNodeWithBinding(page, 'list')).toHaveCount(0)
await expect(await edgesToNodeWithBinding(page, 'final')).toHaveCount(3)
await expect(await edgesToNodeWithBinding(page, 'prod')).toHaveCount(3)
await expect(await edgesToNodeWithBinding(page, 'sum')).toHaveCount(6)
await expect(await edgesToNodeWithBinding(page, 'final')).toHaveCount(EDGE_PARTS)
await expect(await edgesToNodeWithBinding(page, 'prod')).toHaveCount(EDGE_PARTS)
await expect(await edgesToNodeWithBinding(page, 'sum')).toHaveCount(2 * EDGE_PARTS)
await expect(await edgesToNodeWithBinding(page, 'ten')).toHaveCount(0)
await expect(await edgesToNodeWithBinding(page, 'five')).toHaveCount(0)
})
@ -42,18 +43,21 @@ test('Existence of edges between nodes', async ({ page }) => {
test('Hover behaviour of edges', async ({ page }) => {
await actions.goToGraph(page)
// One for interaction, one for rendering, one for the arrow element.
await expect(await edgesFromNodeWithBinding(page, 'ten')).toHaveCount(3)
const edgeElements = await edgesFromNodeWithBinding(page, 'ten')
await expect(edgeElements).toHaveCount(EDGE_PARTS)
const targetEdge = page.locator('path:nth-child(4)')
const targetEdge = edgeElements.first()
await expect(targetEdge).toHaveClass('edge io')
// It is not currently possible to interact with edges in the default node layout.
// See: https://github.com/enso-org/enso/issues/8938
/*
// Hover over edge to the right of node with binding `ten`.
await targetEdge.hover({
position: { x: 65, y: 135.0 },
force: true,
position: { x: 60, y: 45 }, // source node
})
// Expect an extra edge for the split rendering.
const edgeElements = await edgesFromNodeWithBinding(page, 'ten')
await expect(edgeElements).toHaveCount(4)
const hoveredEdgeElements = await edgesFromNodeWithBinding(page, 'ten')
await expect(hoveredEdgeElements).toHaveCount(2 * EDGE_PARTS)
// Expect the top edge part to be dimmed
const topEdge = page.locator('path:nth-child(4)')
@ -61,4 +65,5 @@ test('Hover behaviour of edges', async ({ page }) => {
// Expect the bottom edge part not to be dimmed
const bottomEdge = page.locator('path:nth-child(6)')
await expect(bottomEdge).toHaveClass('edge visible')
*/
})

2
app/gui2/eslint.config.js
View File

@ -9,7 +9,7 @@ const DIR_NAME = path.dirname(url.fileURLToPath(import.meta.url))
const conf = [
{
ignores: ['rust-ffi/pkg', 'dist', 'src/generated', 'templates'],
ignores: ['rust-ffi/pkg', 'rust-ffi/node-pkg', 'dist', 'src/generated', 'templates'],
},
...compat.extends('plugin:vue/vue3-recommended'),
eslintJs.configs.recommended,

7
app/gui2/mock/index.ts
View File

@ -4,6 +4,7 @@ import { useGraphStore } from '@/stores/graph'
import { GraphDb, mockNode } from '@/stores/graph/graphDatabase'
import { useProjectStore } from '@/stores/project'
import { ComputedValueRegistry } from '@/stores/project/computedValueRegistry'
import { Ast } from '@/util/ast'
import { MockTransport, MockWebSocket } from '@/util/net'
import { getActivePinia } from 'pinia'
import { ref, type App } from 'vue'
@ -65,7 +66,11 @@ export function projectStore() {
const projectStore = useProjectStore(getActivePinia())
const mod = projectStore.projectModel.createNewModule('Main.enso')
mod.doc.ydoc.emit('load', [])
mod.doc.setCode('main =\n')
const syncModule = new Ast.MutableModule(mod.doc.ydoc)
mod.transact(() => {
const root = Ast.parseBlock('main =\n', syncModule)
syncModule.replaceRoot(root)
})
return projectStore
}

10
app/gui2/package.json
View File

@ -23,10 +23,11 @@
"typecheck": "vue-tsc --noEmit -p tsconfig.app.json --composite false",
"lint": "eslint .",
"format": "prettier --version && prettier --write src/ && eslint . --fix",
"build-rust-ffi": "wasm-pack build ./rust-ffi --release --target web",
"generate-ast-schema": "cargo run -p enso-parser-schema > src/generated/ast-schema.json",
"generate-ast-types": "tsx ./parser-codegen/index.ts src/generated/ast-schema.json src/generated/ast.ts",
"preinstall": "npm run build-rust-ffi && npm run generate-ast-schema && npm run generate-ast-types && npm run generate-metadata && npm run download-fonts",
"clean-old-generated-directory": "rimraf src/generated",
"build-rust-ffi": "wasm-pack build ./rust-ffi --release --target web && wasm-pack build ./rust-ffi --out-dir node-pkg --target nodejs",
"generate-ast-schema": "cargo run -p enso-parser-schema > shared/ast/generated/ast-schema.json",
"generate-ast-types": "tsx ./parser-codegen/index.ts shared/ast/generated/ast-schema.json shared/ast/generated/ast.ts",
"preinstall": "npm run clean-old-generated-directory && npm run build-rust-ffi && npm run generate-ast-schema && npm run generate-ast-types && npm run generate-metadata && npm run download-fonts",
"postinstall": "playwright install",
"generate-metadata": "node scripts/generateIconMetadata.js",
"download-fonts": "node scripts/downloadFonts.js"
@ -64,6 +65,7 @@
"pinia": "^2.1.6",
"postcss-inline-svg": "^6.0.0",
"postcss-nesting": "^12.0.1",
"rimraf": "^5.0.5",
"semver": "^7.5.4",
"sucrase": "^3.34.0",
"vue": "^3.3.4",

2
app/gui2/parser-codegen/codegen.ts
View File

@ -60,7 +60,7 @@ export function implement(schema: Schema.Schema): string {
),
),
),
tsf.createStringLiteral('@/util/parserSupport', true),
tsf.createStringLiteral('../parserSupport', true),
undefined,
),
)

22
app/gui2/shared/ast/ffi.ts Normal file
View File

@ -0,0 +1,22 @@
import init, {
is_ident_or_operator,
parse_doc_to_json,
parse as parse_tree,
} from '../../rust-ffi/pkg/rust_ffi'
import { isNode } from '../util/detect'
export async function initializeFFI(path?: string | undefined) {
if (isNode) {
const fs = await import('node:fs/promises')
const buffer = fs.readFile(path ?? './rust-ffi/pkg/rust_ffi_bg.wasm')
await init(buffer)
} else {
await init()
}
}
// TODO[ao]: We cannot to that, because the ffi is used by cjs modules.
// await initializeFFI()
// eslint-disable-next-line camelcase
export { is_ident_or_operator, parse_doc_to_json, parse_tree }

0
app/gui2/src/generated/.gitkeep → app/gui2/shared/ast/generated/.gitkeep
View File

70
app/gui2/shared/ast/index.ts Normal file
View File

@ -0,0 +1,70 @@
import * as random from 'lib0/random'
import type { ExternalId } from '../yjsModel'
import type { Module } from './mutableModule'
import type { SyncTokenId } from './token'
import type { AstId } from './tree'
import { Ast, MutableAst } from './tree'
export * from './mutableModule'
export * from './parse'
export * from './token'
export * from './tree'
declare const brandOwned: unique symbol
/** Used to mark references required to be unique.
*
* Note that the typesystem cannot stop you from copying an `Owned`,
* but that is an easy mistake to see (because it occurs locally).
*
* We can at least require *obtaining* an `Owned`,
* which statically prevents the otherwise most likely usage errors when rearranging ASTs.
*/
export type Owned<T = MutableAst> = T & { [brandOwned]: never }
/** @internal */
export function asOwned<T>(t: T): Owned<T> {
return t as Owned<T>
}
export type NodeChild<T = AstId | SyncTokenId> = { whitespace?: string | undefined; node: T }
export function newExternalId(): ExternalId {
return random.uuidv4() as ExternalId
}
/** @internal */
export function parentId(ast: Ast): AstId | undefined {
return ast.fields.get('parent')
}
/** Returns the given IDs, and the IDs of all their ancestors. */
export function subtrees(module: Module, ids: Iterable<AstId>) {
const subtrees = new Set<AstId>()
for (const id of ids) {
let ast = module.get(id)
while (ast != null && !subtrees.has(ast.id)) {
subtrees.add(ast.id)
ast = ast.parent()
}
}
return subtrees
}
/** Returns the IDs of the ASTs that are not descendants of any others in the given set. */
export function subtreeRoots(module: Module, ids: Set<AstId>) {
const roots = new Array<AstId>()
for (const id of ids) {
const astInModule = module.get(id)
if (!astInModule) continue
let ast = astInModule.parent()
let hasParentInSet
while (ast != null) {
if (ids.has(ast.id)) {
hasParentInSet = true
break
}
ast = ast.parent()
}
if (!hasParentInSet) roots.push(id)
}
return roots
}

364
app/gui2/shared/ast/mutableModule.ts Normal file
View File

@ -0,0 +1,364 @@
import * as random from 'lib0/random'
import * as Y from 'yjs'
import type { AstId, Owned, SyncTokenId } from '.'
import { Token, asOwned, isTokenId, newExternalId } from '.'
import { assert, assertDefined } from '../util/assert'
import type { ExternalId, SourceRange } from '../yjsModel'
import type { AstFields, FixedMap, Mutable } from './tree'
import {
Ast,
Invalid,
MutableAst,
MutableInvalid,
Wildcard,
invalidFields,
materializeMutable,
setAll,
} from './tree'
export interface Module {
edit(): MutableModule
root(): Ast | undefined
get(id: AstId): Ast | undefined
get(id: AstId | undefined): Ast | undefined
/////////////////////////////////
checkedGet(id: AstId): Ast
checkedGet(id: AstId | undefined): Ast | undefined
getToken(token: SyncTokenId): Token
getToken(token: SyncTokenId | undefined): Token | undefined
getAny(node: AstId | SyncTokenId): Ast | Token
has(id: AstId): boolean
getSpan(id: AstId): SourceRange | undefined
}
export interface ModuleUpdate {
nodesAdded: AstId[]
nodesDeleted: AstId[]
fieldsUpdated: { id: AstId; fields: (readonly [string, unknown])[] }[]
metadataUpdated: { id: AstId; changes: Map<string, unknown> }[]
}
type YNode = FixedMap<AstFields>
type YNodes = Y.Map<YNode>
export class MutableModule implements Module {
private readonly nodes: YNodes
get ydoc() {
const ydoc = this.nodes.doc
assert(ydoc != null)
return ydoc
}
/** Return this module's copy of `ast`, if this module was created by cloning `ast`'s module. */
getVersion<T extends Ast>(ast: T): Mutable<T> {
const instance = this.checkedGet(ast.id)
return instance as Mutable<T>
}
edit(): MutableModule {
const doc = new Y.Doc()
Y.applyUpdateV2(doc, Y.encodeStateAsUpdateV2(this.ydoc))
return new MutableModule(doc)
}
root(): MutableAst | undefined {
return this.rootPointer()?.expression
}
replaceRoot(newRoot: Owned | undefined): Owned | undefined {
if (newRoot) {
const rootPointer = this.rootPointer()
if (rootPointer) {
return rootPointer.expression.replace(newRoot)
} else {
invalidFields(this, this.baseObject('Invalid', undefined, ROOT_ID), {
whitespace: '',
node: newRoot,
})
return undefined
}
} else {
const oldRoot = this.root()
if (!oldRoot) return
this.nodes.delete(ROOT_ID)
oldRoot.fields.set('parent', undefined)
return asOwned(oldRoot)
}
}
syncRoot(root: Owned) {
this.replaceRoot(root)
this.gc()
}
private gc() {
const live = new Set<AstId>()
const active = new Array<Ast>()
let next: Ast | undefined = this.root()
while (next) {
for (const child of next.children()) {
if (child instanceof Ast) active.push(child)
}
live.add(next.id)
next = active.pop()
}
const all = Array.from(this.nodes.keys())
for (const id of all) {
if (id === ROOT_ID) continue
assert(isAstId(id))
if (!live.has(id)) this.nodes.delete(id)
}
}
/** Copy the given node into the module. */
copy<T extends Ast>(ast: T): Owned<Mutable<T>> {
const id = newAstId(ast.typeName())
const fields = ast.fields.clone()
this.nodes.set(id, fields as any)
fields.set('id', id)
fields.set('parent', undefined)
const ast_ = materializeMutable(this, fields)
ast_.importReferences(ast.module)
return ast_ as Owned<Mutable<typeof ast>>
}
static Transient() {
return new this(new Y.Doc())
}
observe(observer: (update: ModuleUpdate) => void) {
this.nodes.observeDeep((events) => observer(this.observeEvents(events)))
}
getStateAsUpdate(): ModuleUpdate {
const updateBuilder = new UpdateBuilder(this.nodes)
for (const id of this.nodes.keys()) updateBuilder.addNode(id as AstId)
return updateBuilder
}
applyUpdate(update: Uint8Array, origin?: string): ModuleUpdate | undefined {
let summary: ModuleUpdate | undefined
const observer = (events: Y.YEvent<any>[]) => {
summary = this.observeEvents(events)
}
this.nodes.observeDeep(observer)
Y.applyUpdate(this.ydoc, update, origin)
this.nodes.unobserveDeep(observer)
return summary
}
private observeEvents(events: Y.YEvent<any>[]): ModuleUpdate {
const updateBuilder = new UpdateBuilder(this.nodes)
for (const event of events) {
if (event.target === this.nodes) {
// Updates to the node map.
for (const [key, change] of event.changes.keys) {
const id = key as AstId
switch (change.action) {
case 'add':
updateBuilder.addNode(id)
break
case 'update':
updateBuilder.updateAllFields(id)
break
case 'delete':
updateBuilder.deleteNode(id)
break
}
}
} else if (event.target.parent === this.nodes) {
// Updates to a node's fields.
assert(event.target instanceof Y.Map)
const id = event.target.get('id') as AstId
const node = this.nodes.get(id)
if (!node) continue
const changes: (readonly [string, unknown])[] = Array.from(event.changes.keys, ([key]) => [
key,
node.get(key as any),
])
updateBuilder.updateFields(id, changes)
} else if (event.target.parent.parent === this.nodes) {
// Updates to fields of a metadata object within a node.
const id = event.target.parent.get('id') as AstId
const node = this.nodes.get(id)
if (!node) continue
const metadata = node.get('metadata') as unknown as Map<string, unknown>
const changes: (readonly [string, unknown])[] = Array.from(event.changes.keys, ([key]) => [
key,
metadata.get(key as any),
])
updateBuilder.updateMetadata(id, changes)
}
}
return updateBuilder
}
clear() {
this.nodes.clear()
}
checkedGet(id: AstId): Mutable
checkedGet(id: AstId | undefined): Mutable | undefined
checkedGet(id: AstId | undefined): Mutable | undefined {
if (!id) return undefined
const ast = this.get(id)
assert(ast !== undefined, 'id in module')
return ast
}
get(id: AstId): Mutable | undefined
get(id: AstId | undefined): Mutable | undefined
get(id: AstId | undefined): Mutable | undefined {
if (!id) return undefined
const nodeData = this.nodes.get(id)
if (!nodeData) return undefined
const fields = nodeData as any
return materializeMutable(this, fields)
}
replace(id: AstId, value: Owned): Owned | undefined {
return this.get(id)?.replace(value)
}
replaceValue(id: AstId, value: Owned): Owned | undefined {
return this.get(id)?.replaceValue(value)
}
take(id: AstId): Owned {
return this.replace(id, Wildcard.new(this)) || asOwned(this.checkedGet(id))
}
updateValue<T extends MutableAst>(id: AstId, f: (x: Owned) => Owned<T>): T | undefined {
return this.get(id)?.updateValue(f)
}
/////////////////////////////////////////////
getSpan(id: AstId) {
return undefined
}
constructor(doc: Y.Doc) {
this.nodes = doc.getMap<YNode>('nodes')
}
private rootPointer(): MutableRootPointer | undefined {
const rootPointer = this.get(ROOT_ID)
if (rootPointer) return rootPointer as MutableRootPointer
}
/** @internal */
baseObject(type: string, externalId?: ExternalId, overrideId?: AstId): FixedMap<AstFields> {
const map = new Y.Map<unknown>()
const map_ = map as unknown as FixedMap<{}>
const id = overrideId ?? newAstId(type)
const metadata = new Y.Map<unknown>() as unknown as FixedMap<{}>
const metadataFields = setAll(metadata, {
externalId: externalId ?? newExternalId(),
})
const fields = setAll(map_, {
id,
type: type,
parent: undefined,
metadata: metadataFields,
})
this.nodes.set(id, fields)
return fields
}
/** @internal */
getToken(token: SyncTokenId): Token
getToken(token: SyncTokenId | undefined): Token | undefined
getToken(token: SyncTokenId | undefined): Token | undefined {
if (!token) return token
if (token instanceof Token) return token
return Token.withId(token.code_, token.tokenType_, token.id)
}
getAny(node: AstId | SyncTokenId): MutableAst | Token {
return isTokenId(node) ? this.getToken(node) : this.checkedGet(node)
}
/** @internal Copy a node into the module, if it is bound to a different module. */
copyIfForeign<T extends MutableAst>(ast: Owned<T>): Owned<T>
copyIfForeign<T extends MutableAst>(ast: Owned<T> | undefined): Owned<T> | undefined {
if (!ast) return ast
if (ast.module === this) return ast
return this.copy(ast) as any
}
/** @internal */
delete(id: AstId) {
this.nodes.delete(id)
}
/** @internal */
has(id: AstId) {
return this.nodes.has(id)
}
}
type MutableRootPointer = MutableInvalid & { get expression(): MutableAst | undefined }
/** @internal */
export interface RootPointer extends Invalid {}
function newAstId(type: string): AstId {
return `ast:${type}#${random.uint53()}` as AstId
}
/** Checks whether the input looks like an AstId. */
export function isAstId(value: string): value is AstId {
return /ast:[A-Za-z]*#[0-9]*/.test(value)
}
export const ROOT_ID = `Root` as AstId
class UpdateBuilder implements ModuleUpdate {
readonly nodesAdded: AstId[] = []
readonly nodesDeleted: AstId[] = []
readonly fieldsUpdated: { id: AstId; fields: (readonly [string, unknown])[] }[] = []
readonly metadataUpdated: { id: AstId; changes: Map<string, unknown> }[] = []
private readonly nodes: YNodes
constructor(nodes: YNodes) {
this.nodes = nodes
}
addNode(id: AstId) {
this.nodesAdded.push(id)
this.updateAllFields(id)
}
updateAllFields(id: AstId) {
this.updateFields(id, this.nodes.get(id)!.entries())
}
updateFields(id: AstId, changes: Iterable<readonly [string, unknown]>) {
const fields = new Array<readonly [string, unknown]>()
let metadataChanges = undefined
for (const entry of changes) {
const [key, value] = entry
if (key === 'metadata') {
assert(value instanceof Y.Map)
metadataChanges = new Map<string, unknown>(value.entries())
} else {
assert(!(value instanceof Y.AbstractType))
fields.push(entry)
}
}
if (fields.length !== 0) this.fieldsUpdated.push({ id, fields })
if (metadataChanges) this.metadataUpdated.push({ id, changes: metadataChanges })
}
updateMetadata(id: AstId, changes: Iterable<readonly [string, unknown]>) {
const changeMap = new Map<string, unknown>()
for (const [key, value] of changes) changeMap.set(key, value)
this.metadataUpdated.push({ id, changes: changeMap })
}
deleteNode(id: AstId) {
this.nodesDeleted.push(id)
}
}

514
app/gui2/shared/ast/parse.ts Normal file
View File

@ -0,0 +1,514 @@
import * as map from 'lib0/map'
import type { AstId, NodeChild, Owned } from '.'
import { Token, asOwned, parentId, subtreeRoots } from '.'
import { assert, assertDefined, assertEqual } from '../util/assert'
import type { SourceRange, SourceRangeKey } from '../yjsModel'
import { IdMap, isUuid, sourceRangeFromKey, sourceRangeKey } from '../yjsModel'
import { parse_tree } from './ffi'
import * as RawAst from './generated/ast'
import { MutableModule } from './mutableModule'
import type { LazyObject } from './parserSupport'
import {
App,
Assignment,
Ast,
BodyBlock,
Documented,
Function,
Generic,
Group,
Ident,
Import,
Invalid,
MutableBodyBlock,
MutableIdent,
NegationApp,
NumericLiteral,
OprApp,
PropertyAccess,
TextLiteral,
UnaryOprApp,
Wildcard,
} from './tree'
export function parseEnso(code: string): RawAst.Tree {
const blob = parse_tree(code)
return RawAst.Tree.read(new DataView(blob.buffer), blob.byteLength - 4)
}
export function normalize(rootIn: Ast): Ast {
const printed = print(rootIn)
const idMap = spanMapToIdMap(printed.info)
const module = MutableModule.Transient()
const tree = parseEnso(printed.code)
const { root: parsed, spans } = abstract(module, tree, printed.code)
module.replaceRoot(parsed)
setExternalIds(module, spans, idMap)
return parsed
}
export function abstract(
module: MutableModule,
tree: RawAst.Tree,
code: string,
): { root: Owned; spans: SpanMap; toRaw: Map<AstId, RawAst.Tree> } {
const tokens = new Map()
const nodes = new Map()
const toRaw = new Map()
const root = abstractTree(module, tree, code, nodes, tokens, toRaw).node
const spans = { tokens, nodes }
return { root, spans, toRaw }
}
function abstractTree(
module: MutableModule,
tree: RawAst.Tree,
code: string,
nodesOut: NodeSpanMap,
tokensOut: TokenSpanMap,
toRaw: Map<AstId, RawAst.Tree>,
): { whitespace: string | undefined; node: Owned } {
const recurseTree = (tree: RawAst.Tree) =>
abstractTree(module, tree, code, nodesOut, tokensOut, toRaw)
const recurseToken = (token: RawAst.Token.Token) => abstractToken(token, code, tokensOut)
const visitChildren = (tree: LazyObject) => {
const children: NodeChild<Owned | Token>[] = []
const visitor = (child: LazyObject) => {
if (RawAst.Tree.isInstance(child)) {
children.push(recurseTree(child))
} else if (RawAst.Token.isInstance(child)) {
children.push(recurseToken(child))
} else {
child.visitChildren(visitor)
}
}
tree.visitChildren(visitor)
return children
}
const whitespaceStart = tree.whitespaceStartInCodeParsed
const whitespaceEnd = whitespaceStart + tree.whitespaceLengthInCodeParsed
const whitespace = code.substring(whitespaceStart, whitespaceEnd)
const codeStart = whitespaceEnd
const codeEnd = codeStart + tree.childrenLengthInCodeParsed
const spanKey = nodeKey(codeStart, codeEnd - codeStart)
let node: Owned
switch (tree.type) {
case RawAst.Tree.Type.BodyBlock: {
const lines = Array.from(tree.statements, (line) => {
const newline = recurseToken(line.newline)
const expression = line.expression ? recurseTree(line.expression) : undefined
return { newline, expression }
})
node = BodyBlock.concrete(module, lines)
break
}
case RawAst.Tree.Type.Function: {
const name = recurseTree(tree.name)
const argumentDefinitions = Array.from(tree.args, (arg) => visitChildren(arg))
const equals = recurseToken(tree.equals)
const body = tree.body !== undefined ? recurseTree(tree.body) : undefined
node = Function.concrete(module, name, argumentDefinitions, equals, body)
break
}
case RawAst.Tree.Type.Ident: {
const token = recurseToken(tree.token)
node = Ident.concrete(module, token)
break
}
case RawAst.Tree.Type.Assignment: {
const pattern = recurseTree(tree.pattern)
const equals = recurseToken(tree.equals)
const value = recurseTree(tree.expr)
node = Assignment.concrete(module, pattern, equals, value)
break
}
case RawAst.Tree.Type.App: {
const func = recurseTree(tree.func)
const arg = recurseTree(tree.arg)
node = App.concrete(module, func, undefined, undefined, arg)
break
}
case RawAst.Tree.Type.NamedApp: {
const func = recurseTree(tree.func)
const open = tree.open ? recurseToken(tree.open) : undefined
const name = recurseToken(tree.name)
const equals = recurseToken(tree.equals)
const arg = recurseTree(tree.arg)
const close = tree.close ? recurseToken(tree.close) : undefined
const parens = open && close ? { open, close } : undefined
const nameSpecification = { name, equals }
node = App.concrete(module, func, parens, nameSpecification, arg)
break
}
case RawAst.Tree.Type.UnaryOprApp: {
const opr = recurseToken(tree.opr)
const arg = tree.rhs ? recurseTree(tree.rhs) : undefined
if (arg && opr.node.code() === '-') {
node = NegationApp.concrete(module, opr, arg)
} else {
node = UnaryOprApp.concrete(module, opr, arg)
}
break
}
case RawAst.Tree.Type.OprApp: {
const lhs = tree.lhs ? recurseTree(tree.lhs) : undefined
const opr = tree.opr.ok
? [recurseToken(tree.opr.value)]
: Array.from(tree.opr.error.payload.operators, recurseToken)
const rhs = tree.rhs ? recurseTree(tree.rhs) : undefined
if (opr.length === 1 && opr[0]?.node.code() === '.' && rhs?.node instanceof MutableIdent) {
// Propagate type.
const rhs_ = { ...rhs, node: rhs.node }
node = PropertyAccess.concrete(module, lhs, opr[0], rhs_)
} else {
node = OprApp.concrete(module, lhs, opr, rhs)
}
break
}
case RawAst.Tree.Type.Number: {
const tokens = []
if (tree.base) tokens.push(recurseToken(tree.base))
if (tree.integer) tokens.push(recurseToken(tree.integer))
if (tree.fractionalDigits) {
tokens.push(recurseToken(tree.fractionalDigits.dot))
tokens.push(recurseToken(tree.fractionalDigits.digits))
}
node = NumericLiteral.concrete(module, tokens)
break
}
case RawAst.Tree.Type.Wildcard: {
const token = recurseToken(tree.token)
node = Wildcard.concrete(module, token)
break
}
// These expression types are (or will be) used for backend analysis.
// The frontend can ignore them, avoiding some problems with expressions sharing spans
// (which makes it impossible to give them unique IDs in the current IdMap format).
case RawAst.Tree.Type.OprSectionBoundary:
case RawAst.Tree.Type.TemplateFunction:
return { whitespace, node: recurseTree(tree.ast).node }
case RawAst.Tree.Type.Invalid: {
const expression = recurseTree(tree.ast)
node = Invalid.concrete(module, expression)
break
}
case RawAst.Tree.Type.Group: {
const open = tree.open ? recurseToken(tree.open) : undefined
const expression = tree.body ? recurseTree(tree.body) : undefined
const close = tree.close ? recurseToken(tree.close) : undefined
node = Group.concrete(module, open, expression, close)
break
}
case RawAst.Tree.Type.TextLiteral: {
const open = tree.open ? recurseToken(tree.open) : undefined
const newline = tree.newline ? recurseToken(tree.newline) : undefined
const elements = []
for (const e of tree.elements) {
elements.push(...visitChildren(e))
}
const close = tree.close ? recurseToken(tree.close) : undefined
node = TextLiteral.concrete(module, open, newline, elements, close)
break
}
case RawAst.Tree.Type.Documented: {
const open = recurseToken(tree.documentation.open)
const elements = []
for (const e of tree.documentation.elements) {
elements.push(...visitChildren(e))
}
const newlines = Array.from(tree.documentation.newlines, recurseToken)
const expression = tree.expression ? recurseTree(tree.expression) : undefined
node = Documented.concrete(module, open, elements, newlines, expression)
break
}
case RawAst.Tree.Type.Import: {
const recurseBody = (tree: RawAst.Tree) => {
const body = recurseTree(tree)
if (body.node instanceof Invalid && body.node.code() === '') return undefined
return body
}
const recurseSegment = (segment: RawAst.MultiSegmentAppSegment) => ({
header: recurseToken(segment.header),
body: segment.body ? recurseBody(segment.body) : undefined,
})
const polyglot = tree.polyglot ? recurseSegment(tree.polyglot) : undefined
const from = tree.from ? recurseSegment(tree.from) : undefined
const import_ = recurseSegment(tree.import)
const all = tree.all ? recurseToken(tree.all) : undefined
const as = tree.as ? recurseSegment(tree.as) : undefined
const hiding = tree.hiding ? recurseSegment(tree.hiding) : undefined
node = Import.concrete(module, polyglot, from, import_, all, as, hiding)
break
}
default: {
node = Generic.concrete(module, visitChildren(tree))
}
}
toRaw.set(node.id, tree)
map.setIfUndefined(nodesOut, spanKey, (): Ast[] => []).unshift(node)
return { node, whitespace }
}
function abstractToken(
token: RawAst.Token,
code: string,
tokensOut: TokenSpanMap,
): { whitespace: string; node: Token } {
const whitespaceStart = token.whitespaceStartInCodeBuffer
const whitespaceEnd = whitespaceStart + token.whitespaceLengthInCodeBuffer
const whitespace = code.substring(whitespaceStart, whitespaceEnd)
const codeStart = token.startInCodeBuffer
const codeEnd = codeStart + token.lengthInCodeBuffer
const tokenCode = code.substring(codeStart, codeEnd)
const key = tokenKey(codeStart, codeEnd - codeStart)
const node = Token.new(tokenCode, token.type)
tokensOut.set(key, node)
return { whitespace, node }
}
declare const nodeKeyBrand: unique symbol
export type NodeKey = SourceRangeKey & { [nodeKeyBrand]: never }
declare const tokenKeyBrand: unique symbol
export type TokenKey = SourceRangeKey & { [tokenKeyBrand]: never }
export function nodeKey(start: number, length: number): NodeKey {
return sourceRangeKey([start, start + length]) as NodeKey
}
export function tokenKey(start: number, length: number): TokenKey {
return sourceRangeKey([start, start + length]) as TokenKey
}
export type NodeSpanMap = Map<NodeKey, Ast[]>
export type TokenSpanMap = Map<TokenKey, Token>
export interface SpanMap {
nodes: NodeSpanMap
tokens: TokenSpanMap
}
interface PrintedSource {
info: SpanMap
code: string
}
export function spanMapToIdMap(spans: SpanMap): IdMap {
const idMap = new IdMap()
for (const [key, token] of spans.tokens.entries()) {
assert(isUuid(token.id))
idMap.insertKnownId(sourceRangeFromKey(key), token.id)
}
for (const [key, asts] of spans.nodes.entries()) {
for (const ast of asts) {
assert(isUuid(ast.externalId))
idMap.insertKnownId(sourceRangeFromKey(key), ast.externalId)
}
}
return idMap
}
export function spanMapToSpanGetter(spans: SpanMap): (id: AstId) => SourceRange | undefined {
const reverseMap = new Map<AstId, SourceRange>()
for (const [key, asts] of spans.nodes) {
for (const ast of asts) {
reverseMap.set(ast.id, sourceRangeFromKey(key))
}
}
return (id) => reverseMap.get(id)
}
/** Return stringification with associated ID map. This is only exported for testing. */
export function print(ast: Ast): PrintedSource {
const info: SpanMap = {
nodes: new Map(),
tokens: new Map(),
}
const code = ast.printSubtree(info, 0, undefined)
return { info, code }
}
/** Parse the input as a block. */
export function parseBlock(code: string, inModule?: MutableModule) {
return parseBlockWithSpans(code, inModule).root
}
/** Parse the input. If it contains a single expression at the top level, return it; otherwise, return a block. */
export function parse(code: string, module?: MutableModule): Owned {
const module_ = module ?? MutableModule.Transient()
const ast = parseBlock(code, module_)
const [expr] = ast.statements()
if (!expr) return ast
const parent = parentId(expr)
if (parent) module_.delete(parent)
expr.fields.set('parent', undefined)
return asOwned(expr)
}
export function parseBlockWithSpans(
code: string,
inModule?: MutableModule,
): { root: Owned<MutableBodyBlock>; spans: SpanMap } {
const tree = parseEnso(code)
const module = inModule ?? MutableModule.Transient()
return fromRaw(tree, code, module)
}
function fromRaw(
tree: RawAst.Tree,
code: string,
inModule?: MutableModule,
): {
root: Owned<MutableBodyBlock>
spans: SpanMap
toRaw: Map<AstId, RawAst.Tree>
} {
const module = inModule ?? MutableModule.Transient()
const ast = abstract(module, tree, code)
const spans = ast.spans
// The root of the tree produced by the parser is always a `BodyBlock`.
const root = ast.root as Owned<MutableBodyBlock>
return { root, spans, toRaw: ast.toRaw }
}
export function parseExtended(code: string, idMap?: IdMap | undefined, inModule?: MutableModule) {
const rawRoot = parseEnso(code)
const module = inModule ?? MutableModule.Transient()
const { root, spans, toRaw, idMapUpdates } = module.ydoc.transact(() => {
const { root, spans, toRaw } = fromRaw(rawRoot, code, module)
root.module.replaceRoot(root)
const idMapUpdates = idMap ? setExternalIds(root.module, spans, idMap) : 0
return { root, spans, toRaw, idMapUpdates }
}, 'local')
const getSpan = spanMapToSpanGetter(spans)
const idMapOut = spanMapToIdMap(spans)
return { root, idMap: idMapOut, getSpan, toRaw, idMapUpdates }
}
export function setExternalIds(edit: MutableModule, spans: SpanMap, ids: IdMap) {
let astsMatched = 0
let idsUnmatched = 0
let asts = 0
edit.root()?.visitRecursiveAst((_ast) => (asts += 1))
for (const [key, externalId] of ids.entries()) {
const asts = spans.nodes.get(key as NodeKey)
if (asts) {
for (const ast of asts) {
astsMatched += 1
const editAst = edit.getVersion(ast)
if (editAst.externalId !== externalId) editAst.setExternalId(externalId)
}
} else {
idsUnmatched += 1
}
}
return edit.root() ? asts - astsMatched : 0
}
function checkSpans(expected: NodeSpanMap, encountered: NodeSpanMap, code: string) {
const lost = new Array<readonly [NodeKey, Ast]>()
for (const [key, asts] of expected) {
const outermostPrinted = asts[0]
if (!outermostPrinted) continue
for (let i = 1; i < asts.length; ++i) assertEqual(asts[i]?.parentId, asts[i - 1]?.id)
const encounteredAsts = encountered.get(key)
if (encounteredAsts === undefined) lost.push([key, outermostPrinted])
}
const lostInline = new Array<Ast>()
const lostBlock = new Array<Ast>()
for (const [key, ast] of lost) {
const [start, end] = sourceRangeFromKey(key)
;(code.substring(start, end).match(/[\r\n]/) ? lostBlock : lostInline).push(ast)
}
return { lostInline, lostBlock }
}
/** If the input tree's concrete syntax has precedence errors (i.e. its expected code would not parse back to the same
* structure), try to fix it. If possible, it will be repaired by inserting parentheses; if that doesn't fix it, the
* affected subtree will be re-synced to faithfully represent the source code the incorrect tree prints to.
*/
export function repair(
root: BodyBlock,
module?: MutableModule,
): { code: string; fixes: MutableModule | undefined } {
// Print the input to see what spans its nodes expect to have in the output.
const printed = print(root)
// Parse the printed output to see what spans actually correspond to nodes in the printed code.
const reparsed = parseBlockWithSpans(printed.code)
// See if any span we expected to be a node isn't; if so, it likely merged with its parent due to wrong precedence.
const { lostInline, lostBlock } = checkSpans(
printed.info.nodes,
reparsed.spans.nodes,
printed.code,
)
if (lostInline.length === 0) {
if (lostBlock.length !== 0) {
console.warn(`repair: Bad block elements, but all inline elements OK?`)
const fixes = module ?? root.module.edit()
resync(lostBlock, printed.info.nodes, reparsed.spans.nodes, fixes)
return { code: printed.code, fixes }
}
return { code: printed.code, fixes: undefined }
}
// Wrap any "lost" nodes in parentheses.
const fixes = module ?? root.module.edit()
for (const ast of lostInline) {
if (ast instanceof Group) continue
fixes.getVersion(ast).update((ast) => Group.new(fixes, ast))
}
// Verify that it's fixed.
const printed2 = print(fixes.getVersion(root))
const reparsed2 = parseBlockWithSpans(printed2.code)
const { lostInline: lostInline2, lostBlock: lostBlock2 } = checkSpans(
printed2.info.nodes,
reparsed2.spans.nodes,
printed2.code,
)
if (lostInline2.length !== 0 || lostBlock2.length !== 0)
resync([...lostInline2, ...lostBlock2], printed2.info.nodes, reparsed2.spans.nodes, fixes)
return { code: printed2.code, fixes }
}
/**
* Replace subtrees in the module to ensure that the module contents are consistent with the module's code.
*
* @param badAsts - ASTs that, if printed, would not parse to exactly their current content.
* @param badSpans - Span map produced by printing the `badAsts` nodes and all their parents.
* @param goodSpans - Span map produced by parsing the code from the module of `badAsts`.
* @param edit - Module to apply the fixes to; must contain all ASTs in `badAsts`.
*/
function resync(
badAsts: Iterable<Ast>,
badSpans: NodeSpanMap,
goodSpans: NodeSpanMap,
edit: MutableModule,
) {
const parentsOfBadSubtrees = new Set<AstId>()
const badAstIds = new Set(Array.from(badAsts, (ast) => ast.id))
for (const id of subtreeRoots(edit, badAstIds)) {
const parent = edit.checkedGet(id)?.parentId
if (parent) parentsOfBadSubtrees.add(parent)
}
const spanOfBadParent = new Array<readonly [AstId, NodeKey]>()
for (const [span, asts] of badSpans) {
for (const ast of asts) {
if (parentsOfBadSubtrees.has(ast.id)) spanOfBadParent.push([ast.id, span])
}
}
// All ASTs in the module of badAsts should have entries in badSpans.
assertEqual(spanOfBadParent.length, parentsOfBadSubtrees.size)
for (const [id, span] of spanOfBadParent) {
const parent = edit.checkedGet(id)
const goodAst = goodSpans.get(span)?.[0]
// The parent of the root of a bad subtree must be a good AST.
assertDefined(goodAst)
parent.replaceValue(edit.copy(goodAst))
}
console.warn(
`repair: Replaced ${parentsOfBadSubtrees.size} subtrees with their reparsed equivalents.`,
parentsOfBadSubtrees,
)
}

8
app/gui2/src/util/parserSupport.ts → app/gui2/shared/ast/parserSupport.ts
View File

@ -1,8 +1,8 @@
/** This file supports the module in `../generated/ast.ts` that is produced by `parser-codegen`. */
/** This file supports the module in `generated/ast.ts` that is produced by `parser-codegen`. */
export { type Result } from '@/util/data/result'
import { bail } from '@/util/assert'
import { Err, Ok, type Result } from '@/util/data/result'
export { type Result } from '../util/data/result'
import { bail } from '../util/assert'
import { Err, Ok, type Result } from '../util/data/result'
export type ObjectVisitor = (object: LazyObject) => boolean | void
export type ObjectAddressVisitor = (view: DataView, address: number) => boolean | void

122
app/gui2/shared/ast/token.ts Normal file
View File

@ -0,0 +1,122 @@
import type { AstId, Owned } from '.'
import { Ast, newExternalId } from '.'
import { assert } from '../util/assert'
import type { ExternalId } from '../yjsModel'
import { isUuid } from '../yjsModel'
import { is_ident_or_operator } from './ffi'
import * as RawAst from './generated/ast'
export function isToken(t: unknown): t is Token {
return t instanceof Token
}
declare const brandTokenId: unique symbol
export type TokenId = ExternalId & { [brandTokenId]: never }
function newTokenId(): TokenId {
return newExternalId() as TokenId
}
/** @internal */
export interface SyncTokenId {
readonly id: TokenId
code_: string
tokenType_: RawAst.Token.Type | undefined
}
export class Token implements SyncTokenId {
readonly id: TokenId
code_: string
tokenType_: RawAst.Token.Type | undefined
private constructor(code: string, type: RawAst.Token.Type | undefined, id: TokenId) {
this.id = id
this.code_ = code
this.tokenType_ = type
}
get externalId(): TokenId {
return this.id
}
static new(code: string, type?: RawAst.Token.Type) {
return new this(code, type, newTokenId())
}
static withId(code: string, type: RawAst.Token.Type | undefined, id: TokenId) {
assert(isUuid(id))
return new this(code, type, id)
}
code(): string {
return this.code_
}
typeName(): string {
if (this.tokenType_) return RawAst.Token.typeNames[this.tokenType_]!
else return 'Raw'
}
}
// We haven't had much need to distinguish token types, but it's useful to know that an identifier token's code is a
// valid string for an identifier.
export interface IdentifierOrOperatorIdentifierToken extends Token {
code(): IdentifierOrOperatorIdentifier
}
export interface IdentifierToken extends Token {
code(): Identifier
}
declare const qualifiedNameBrand: unique symbol
declare const identifierBrand: unique symbol
declare const operatorBrand: unique symbol
/** A string representing a valid qualified name of our language.
*
* In our language, the segments are separated by `.`. All the segments except the last must be lexical identifiers. The
* last may be an identifier or a lexical operator. A single identifier is also a valid qualified name.
*/
export type QualifiedName = string & { [qualifiedNameBrand]: never }
/** A string representing a lexical identifier. */
export type Identifier = string & { [identifierBrand]: never; [qualifiedNameBrand]: never }
/** A string representing a lexical operator. */
export type Operator = string & { [operatorBrand]: never; [qualifiedNameBrand]: never }
/** A string that can be parsed as an identifier in some contexts.
*
* If it is lexically an identifier (see `StrictIdentifier`), it can be used as identifier anywhere.
*
* If it is lexically an operator (see `Operator`), it takes the syntactic role of an identifier if it is the RHS of
* a `PropertyAccess`, or it is the name of a `Function` being defined within a type. In all other cases, it is not
* valid to use a lexical operator as an identifier (rather, it will usually parse as an `OprApp` or `UnaryOprApp`).
*/
export type IdentifierOrOperatorIdentifier = Identifier | Operator
/** Returns true if `code` can be used as an identifier in some contexts.
*
* If it is lexically an identifier (see `isIdentifier`), it can be used as identifier anywhere.
*
* If it is lexically an operator (see `isOperator`), it takes the syntactic role of an identifier if it is the RHS of
* a `PropertyAccess`, or it is the name of a `Function` being defined within a type. In all other cases, it is not
* valid to use a lexical operator as an identifier (rather, it will usually parse as an `OprApp` or `UnaryOprApp`).
*/
export function isIdentifierOrOperatorIdentifier(
code: string,
): code is IdentifierOrOperatorIdentifier {
return is_ident_or_operator(code) !== 0
}
/** Returns true if `code` is lexically an identifier. */
export function isIdentifier(code: string): code is Identifier {
return is_ident_or_operator(code) === 1
}
/** Returns true if `code` is lexically an operator. */
export function isOperator(code: string): code is Operator {
return is_ident_or_operator(code) === 2
}
/** @internal */
export function isTokenId(t: SyncTokenId | AstId | Ast | Owned<Ast> | Owned): t is SyncTokenId {
return typeof t === 'object' && !(t instanceof Ast)
}

2125
app/gui2/shared/ast/tree.ts Normal file
View File

File diff suppressed because it is too large Load Diff

69
app/gui2/shared/util/assert.ts Normal file
View File

@ -0,0 +1,69 @@
export function assertNever(x: never): never {
bail('Unexpected object: ' + JSON.stringify(x))
}
export function assert(condition: boolean, message?: string): asserts condition {
if (!condition) bail(message ? `Assertion failed: ${message}` : 'Assertion failed')
}
/**
* Checks if the given iterable has the specified length and throws an assertion error
* if the lengths do not match.
*
* @param iterable The iterable to check.
* @param length The expected length of the iterable.
* @param message Optional message for the assertion error.
* @return void
* @throws Error Will throw an error if the length does not match.
*
* The first five elements of the iterable will be displayed in the error message
* if the assertion fails. If the iterable contains more than five elements,
* the remaining elements will be represented as '...'.
*/
export function assertLength<T>(iterable: Iterable<T>, length: number, message?: string): void {
const convertedArray = Array.from(iterable)
const messagePrefix = message ? message + ' ' : ''
const elementRepresentation =
convertedArray.length > 5
? `${convertedArray.slice(0, 5).join(', ')},...`
: convertedArray.join(', ')
assert(
convertedArray.length === length,
`${messagePrefix}Expected iterable of length ${length}, got length ${convertedArray.length}. Elements: [${elementRepresentation}]`,
)
}
export function assertEmpty<T>(iterable: Iterable<T>, message?: string): void {
assertLength(iterable, 0, message)
}
export function assertEqual<T>(actual: T, expected: T, message?: string) {
const messagePrefix = message ? message + ' ' : ''
assert(actual === expected, `${messagePrefix}Expected ${expected}, got ${actual}.`)
}
export function assertNotEqual<T>(actual: T, unexpected: T, message?: string) {
const messagePrefix = message ? message + ' ' : ''
assert(actual !== unexpected, `${messagePrefix}Expected not ${unexpected}, got ${actual}.`)
}
export function assertDefined<T>(x: T | undefined, message?: string): asserts x is T {
const messagePrefix = message ? message + ' ' : ''
assert(x !== undefined, `${messagePrefix}Expected value to be defined.`)
}
export function assertUnreachable(): never {
bail('Unreachable code')
}
/**
* Throw an error with provided message.
*
* It is convenient to use at the end of a nullable chain:
* ```ts
* const x = foo?.bar.baz?.() ?? bail('Expected foo.bar.baz to exist')
* ```
*/
export function bail(message: string): never {
throw new Error(message)
}

25
app/gui2/shared/util/data/opt.ts Normal file
View File

@ -0,0 +1,25 @@
/** @file A value that may be `null` or `undefined`. */
/** Optional value type. This is a replacement for `T | null | undefined` that is more
* convenient to use. We do not select a single value to represent "no value", because we are using
* libraries that disagree whether `null` (e.g. Yjs) or `undefined` (e.g. Vue) should be used for
* that purpose. We want to be compatible with both without needless conversions. In our own code,
* we should return `undefined` for "no value", since that is the default value for empty or no
* `return` expression. In order to test whether an `Opt<T>` is defined or not, use `x == null` or
* `isSome` function.
*
* Note: For JSON-serialized data, prefer explicit `null` over `undefined`, since `undefined` is
* not serializable. Alternatively, use optional field syntax (e.g. `{ x?: number }`). */
export type Opt<T> = T | null | undefined
export function isSome<T>(value: Opt<T>): value is T {
return value != null
}
export function isNone(value: Opt<any>): value is null | undefined {
return value == null
}
export function mapOr<T, R>(optional: Opt<T>, fallback: R, mapper: (value: T) => R): R {
return isSome(optional) ? mapper(optional) : fallback
}

85
app/gui2/shared/util/data/result.ts Normal file
View File

@ -0,0 +1,85 @@
/** @file A generic type that can either hold a value representing a successful result,
* or an error. */
import { isSome, type Opt } from './opt'
export type Result<T = undefined, E = string> =
| { ok: true; value: T }
| { ok: false; error: ResultError<E> }
export function Ok<T>(data: T): Result<T, never> {
return { ok: true, value: data }
}
export function Err<E>(error: E): Result<never, E> {
return { ok: false, error: new ResultError(error) }
}
export function okOr<T, E>(data: Opt<T>, error: E): Result<T, E> {
if (isSome(data)) return Ok(data)
else return Err(error)
}
export function unwrap<T, E>(result: Result<T, E>): T {
if (result.ok) return result.value
else throw result.error
}
export function mapOk<T, U, E>(result: Result<T, E>, f: (value: T) => U): Result<U, E> {
if (result.ok) return Ok(f(result.value))
else return result
}
export function isResult(v: unknown): v is Result {
return (
v != null &&
typeof v === 'object' &&
'ok' in v &&
typeof v.ok === 'boolean' &&
('value' in v || ('error' in v && v.error instanceof ResultError))
)
}
export class ResultError<E = string> {
payload: E
context: (() => string)[]
constructor(payload: E) {
this.payload = payload
this.context = []
}
log(preamble: string = 'Error') {
const ctx =
this.context.length > 0 ? `\n${Array.from(this.context, (ctx) => ctx()).join('\n')}` : ''
console.error(`${preamble}: ${this.payload}${ctx}`)
}
}
export function withContext<T, E>(context: () => string, f: () => Result<T, E>): Result<T, E> {
const result = f()
if (result == null) {
throw new Error('withContext: f() returned null or undefined')
}
if (!result.ok) result.error.context.splice(0, 0, context)
return result
}
/**
* Catch promise rejection of provided types and convert them to a Result type.
*/
export function rejectionToResult<ErrorKind extends new (...args: any[]) => any>(
errorKinds: ErrorKind | ErrorKind[],
): <T>(promise: Promise<T>) => Promise<Result<T, InstanceType<ErrorKind>>> {
const errorKindArray = Array.isArray(errorKinds) ? errorKinds : [errorKinds]
return async (promise) => {
try {
return Ok(await promise)
} catch (error) {
for (const errorKind of errorKindArray) {
if (error instanceof errorKind) return Err(error)
}
throw error
}
}
}

0
app/gui2/src/util/detect.ts → app/gui2/shared/util/detect.ts
View File

46
app/gui2/shared/yjsModel.ts
View File

@ -96,39 +96,12 @@ export class DistributedProject {
}
}
export interface NodeMetadata {
x: number
y: number
vis: VisualizationMetadata | null
}
export class ModuleDoc {
ydoc: Y.Doc
metadata: Y.Map<NodeMetadata>
data: Y.Map<any>
nodes: Y.Map<any>
constructor(ydoc: Y.Doc) {
this.ydoc = ydoc
this.metadata = ydoc.getMap('metadata')
this.data = ydoc.getMap('data')
}
setIdMap(map: IdMap) {
const oldMap = new IdMap(this.data.get('idmap') ?? [])
if (oldMap.isEqual(map)) return
this.data.set('idmap', map.entries())
}
getIdMap(): IdMap {
const map = this.data.get('idmap')
return new IdMap(map ?? [])
}
setCode(code: string) {
this.data.set('code', code)
}
getCode(): string {
return this.data.get('code') ?? ''
this.nodes = ydoc.getMap('nodes')
}
}
@ -144,26 +117,13 @@ export class DistributedModule {
constructor(ydoc: Y.Doc) {
this.doc = new ModuleDoc(ydoc)
this.undoManager = new Y.UndoManager([this.doc.data, this.doc.metadata])
this.undoManager = new Y.UndoManager([this.doc.nodes])
}
transact<T>(fn: () => T): T {
return this.doc.ydoc.transact(fn, 'local')
}
updateNodeMetadata(id: ExternalId, meta: Partial<NodeMetadata>): void {
const existing = this.doc.metadata.get(id) ?? { x: 0, y: 0, vis: null }
this.transact(() => this.doc.metadata.set(id, { ...existing, ...meta }))
}
getNodeMetadata(id: ExternalId): NodeMetadata | null {
return this.doc.metadata.get(id) ?? null
}
getIdMap(): IdMap {
return new IdMap(this.doc.data.get('idmap') ?? [])
}
dispose(): void {
this.doc.ydoc.destroy()
}

2
app/gui2/src/App.vue
View File

@ -5,8 +5,8 @@ import { provideGuiConfig } from '@/providers/guiConfig'
import { useSuggestionDbStore } from '@/stores/suggestionDatabase'
import { configValue, type ApplicationConfig, type ApplicationConfigValue } from '@/util/config'
import ProjectView from '@/views/ProjectView.vue'
import { isDevMode } from 'shared/util/detect'
import { computed, onMounted, toRaw } from 'vue'
import { isDevMode } from './util/detect'
const props = defineProps<{
config: ApplicationConfig

3
app/gui2/src/components/ComponentBrowser/__tests__/filtering.test.ts
View File

@ -13,6 +13,9 @@ import {
} from '@/stores/suggestionDatabase/entry'
import { unwrap } from '@/util/data/result'
import { tryQualifiedName, type QualifiedName } from '@/util/qualifiedName'
import { initializeFFI } from 'shared/ast/ffi'
await initializeFFI()
test.each([
{ ...makeModuleMethod('Standard.Base.Data.read'), groupIndex: 0 },

24
app/gui2/src/components/GraphEditor.vue
View File

@ -28,13 +28,12 @@ import type { RequiredImport } from '@/stores/graph/imports'
import { useProjectStore } from '@/stores/project'
import { groupColorVar, useSuggestionDbStore } from '@/stores/suggestionDatabase'
import { bail } from '@/util/assert'
import type { AstId } from '@/util/ast/abstract.ts'
import type { AstId, NodeMetadataFields } from '@/util/ast/abstract'
import { colorFromString } from '@/util/colors'
import { Rect } from '@/util/data/rect'
import { Vec2 } from '@/util/data/vec2'
import * as set from 'lib0/set'
import { toast } from 'react-toastify'
import type { NodeMetadata } from 'shared/yjsModel'
import { computed, onMounted, onScopeDispose, onUnmounted, ref, watch } from 'vue'
import { ProjectManagerEvents } from '../../../ide-desktop/lib/dashboard/src/utilities/ProjectManager'
import { type Usage } from './ComponentBrowser/input'
@ -281,14 +280,19 @@ const graphBindingsHandler = graphBindings.handler({
const collapsedFunctionEnv = environmentForNodes(collapsedNodeIds.values())
// For collapsed function, only selected nodes would affect placement of the output node.
collapsedFunctionEnv.nodeRects = collapsedFunctionEnv.selectedNodeRects
const meta = new Map<AstId, Partial<NodeMetadata>>()
const { position } = collapsedNodePlacement(DEFAULT_NODE_SIZE, currentFunctionEnv)
meta.set(refactoredNodeId, { x: Math.round(position.x), y: -Math.round(position.y) })
edit
.checkedGet(refactoredNodeId)
.mutableNodeMetadata()
.set('position', { x: position.x, y: position.y })
if (outputNodeId != null) {
const { position } = previousNodeDictatedPlacement(DEFAULT_NODE_SIZE, collapsedFunctionEnv)
meta.set(outputNodeId, { x: Math.round(position.x), y: -Math.round(position.y) })
edit
.checkedGet(outputNodeId)
.mutableNodeMetadata()
.set('position', { x: position.x, y: position.y })
}
graphStore.commitEdit(edit, meta)
graphStore.commitEdit(edit)
} catch (err) {
console.log('Error while collapsing, this is not normal.', err)
}
@ -489,7 +493,7 @@ interface ClipboardData {
/** Node data that is copied to the clipboard. Used for serializing and deserializing the node information. */
interface CopiedNode {
expression: string
metadata: NodeMetadata | undefined
metadata: NodeMetadataFields | undefined
}
/** Copy the content of the selected node to the clipboard. */
@ -498,7 +502,11 @@ function copyNodeContent() {
const node = graphStore.db.nodeIdToNode.get(id)
if (!node) return
const content = node.rootSpan.code()
const metadata = projectStore.module?.getNodeMetadata(id) ?? undefined
const nodeMetadata = node.rootSpan.nodeMetadata
const metadata = {
position: nodeMetadata.get('position'),
visualization: nodeMetadata.get('visualization'),
}
const copiedNode: CopiedNode = { expression: content, metadata }
const clipboardData: ClipboardData = { nodes: [copiedNode] }
const jsonItem = new Blob([JSON.stringify(clipboardData)], { type: ENSO_MIME_TYPE })

8
app/gui2/src/components/GraphEditor/GraphEdge.vue
View File

@ -9,8 +9,6 @@ import theme from '@/util/theme'
import { clamp } from '@vueuse/core'
import { computed, ref } from 'vue'
const DEBUG = false
const selection = injectGraphSelection(true)
const navigator = injectGraphNavigator(true)
const graph = useGraphStore()
@ -55,11 +53,7 @@ const targetRect = computed<Rect | undefined>(() => {
const expr = targetExpr.value
if (expr != null && targetNode.value != null && targetNodeRect.value != null) {
const targetRectRelative = graph.getPortRelativeRect(expr)
if (targetRectRelative == null) {
// This seems to happen for some `Ast.Ident` ports while processing updates, but is quickly fixed.
if (DEBUG) console.warn(`No relative rect found for ${expr}.`)
return
}
if (targetRectRelative == null) return
return targetRectRelative.offsetBy(targetNodeRect.value.pos)
} else if (navigator?.sceneMousePos != null) {
return new Rect(navigator.sceneMousePos, Vec2.Zero)

4
app/gui2/src/components/GraphEditor/GraphEdges.vue
View File

@ -35,7 +35,6 @@ const editingEdge: Interaction = {
}
const target = graph.unconnectedEdge.target ?? selection?.hoveredPort
const targetNode = target && graph.getPortNodeId(target)
console.log(source, target, targetNode)
graph.transact(() => {
if (source != null && sourceNode != targetNode) {
if (target == null) {
@ -57,7 +56,7 @@ const editingEdge: Interaction = {
interaction.setWhen(() => graph.unconnectedEdge != null, editingEdge)
function disconnectEdge(target: PortId) {
graph.editScope((edit) => {
graph.commitDirect((edit) => {
if (!graph.updatePortValue(edit, target, undefined)) {
if (isAstId(target)) {
console.warn(`Failed to disconnect edge from port ${target}, falling back to direct edit.`)
@ -77,7 +76,6 @@ function createEdge(source: AstId, target: PortId) {
const sourceNode = graph.db.getPatternExpressionNodeId(source)
const targetNode = graph.getPortNodeId(target)
if (sourceNode == null || targetNode == null) {
console.log(sourceNode, targetNode, source, target)
return console.error(`Failed to connect edge, source or target node not found.`)
}

3
app/gui2/src/components/GraphEditor/NodeWidgetTree.vue
View File

@ -7,8 +7,6 @@ import { useGraphStore } from '@/stores/graph'
import { Ast } from '@/util/ast'
import { computed, toRef } from 'vue'
const DEBUG = false
const props = defineProps<{ ast: Ast.Ast }>()
const graph = useGraphStore()
const rootPort = computed(() => {
@ -33,7 +31,6 @@ const observedLayoutTransitions = new Set([
])
function handleWidgetUpdates(update: WidgetUpdate) {
if (DEBUG) console.log('Widget Update: ', update)
if (update.portUpdate) {
const {
edit,

14
app/gui2/src/components/GraphEditor/collapsing.ts
View File

@ -204,14 +204,8 @@ export function performCollapse(
outputNodeId = asNodeId(ident.id)
}
const argNames = info.extracted.inputs
const collapsedFunction = Ast.Function.fromStatements(
edit,
collapsedName,
argNames,
collapsed,
true,
)
topLevel.insert(posToInsert, collapsedFunction)
const collapsedFunction = Ast.Function.fromStatements(edit, collapsedName, argNames, collapsed)
topLevel.insert(posToInsert, collapsedFunction, undefined)
return { refactoredNodeId, collapsedNodeIds, outputNodeId }
}
@ -244,6 +238,8 @@ function findInsertionPos(topLevel: Ast.BodyBlock, currentMethodName: string): n
if (import.meta.vitest) {
const { test, expect } = import.meta.vitest
const { initializeFFI } = await import('shared/ast/ffi')
await initializeFFI()
function setupGraphDb(code: string, graphDb: GraphDb) {
const { root, toRaw, getSpan } = Ast.parseExtended(code)
@ -252,7 +248,7 @@ if (import.meta.vitest) {
assert(func instanceof Ast.Function)
const rawFunc = toRaw.get(func.id)
assert(rawFunc?.type === RawAst.Tree.Type.Function)
graphDb.readFunctionAst(func, rawFunc, code, (_id) => undefined, getSpan)
graphDb.readFunctionAst(func, rawFunc, code, getSpan, new Set())
}
interface TestCase {

4
app/gui2/src/main.ts
View File

@ -1,9 +1,10 @@
import { baseConfig, configValue, mergeConfig } from '@/util/config'
import { isDevMode } from '@/util/detect'
import { urlParams } from '@/util/urlParams'
import { isOnLinux } from 'enso-common/src/detect'
import * as dashboard from 'enso-dashboard'
import 'enso-dashboard/src/tailwind.css'
import { initializeFFI } from 'shared/ast/ffi'
import { isDevMode } from 'shared/util/detect'
const INITIAL_URL_KEY = `Enso-initial-url`
const SCAM_WARNING_TIMEOUT = 1000
@ -50,6 +51,7 @@ export interface StringConfig {
}
async function runApp(config: StringConfig | null, accessToken: string | null, metadata?: object) {
await initializeFFI()
running = true
const { mountProjectApp } = await import('./createApp')
if (!running) return

2
app/gui2/src/stores/graph/__tests__/graphDatabase.test.ts
View File

@ -41,7 +41,7 @@ test('Reading graph from definition', () => {
assert(func instanceof Ast.Function)
const rawFunc = toRaw.get(func.id)
assert(rawFunc?.type === RawAst.Tree.Type.Function)
db.readFunctionAst(func, rawFunc, code, (_) => ({ x: 0.0, y: 0.0, vis: null }), getSpan)
db.readFunctionAst(func, rawFunc, code, getSpan, new Set())
const idFromExternal = new Map()
ast.visitRecursiveAst((ast) => idFromExternal.set(ast.externalId, ast.id))

51
app/gui2/src/stores/graph/graphDatabase.ts
View File

@ -3,7 +3,8 @@ import { SuggestionDb, groupColorStyle, type Group } from '@/stores/suggestionDa
import type { SuggestionEntry } from '@/stores/suggestionDatabase/entry'
import { assert } from '@/util/assert'
import { Ast, RawAst } from '@/util/ast'
import type { AstId } from '@/util/ast/abstract'
import type { AstId, NodeMetadata } from '@/util/ast/abstract'
import { subtrees } from '@/util/ast/abstract'
import { AliasAnalyzer } from '@/util/ast/aliasAnalysis'
import { nodeFromAst } from '@/util/ast/node'
import { colorFromString } from '@/util/colors'
@ -20,7 +21,6 @@ import {
sourceRangeKey,
visMetadataEquals,
type ExternalId,
type NodeMetadata,
type SourceRange,
type VisualizationMetadata,
} from 'shared/yjsModel'
@ -130,6 +130,7 @@ export class GraphDb {
private readonly idToExternalMap = reactive(new Map<Ast.AstId, ExternalId>())
private readonly idFromExternalMap = reactive(new Map<ExternalId, Ast.AstId>())
private bindings = new BindingsDb()
private currentFunction: AstId | undefined = undefined
constructor(
private suggestionDb: SuggestionDb,
@ -328,31 +329,37 @@ export class GraphDb {
functionAst_: Ast.Function,
rawFunction: RawAst.Tree.Function,
moduleCode: string,
getMeta: (id: ExternalId) => NodeMetadata | undefined,
getSpan: (id: AstId) => SourceRange | undefined,
dirtyNodes: Set<AstId>,
) {
const functionChanged = functionAst_.id !== this.currentFunction
const knownDirtySubtrees = functionChanged ? null : subtrees(functionAst_.module, dirtyNodes)
const subtreeDirty = (id: AstId) => !knownDirtySubtrees || knownDirtySubtrees.has(id)
this.currentFunction = functionAst_.id
const currentNodeIds = new Set<NodeId>()
for (const nodeAst of functionAst_.bodyExpressions()) {
const newNode = nodeFromAst(nodeAst)
const nodeId = asNodeId(newNode.rootSpan.id)
const node = this.nodeIdToNode.get(nodeId)
const externalId = this.idToExternal(nodeId)
const nodeMeta = externalId && getMeta(externalId)
const nodeMeta = (node ?? newNode).rootSpan.nodeMetadata
currentNodeIds.add(nodeId)
if (node == null) {
// We are notified of new or changed metadata by `updateMetadata`, so we only need to read existing metadata
// when we switch to a different function.
if (functionChanged) {
const pos = nodeMeta.get('position') ?? { x: 0, y: 0 }
newNode.position = new Vec2(pos.x, pos.y)
newNode.vis = nodeMeta.get('visualization')
}
this.nodeIdToNode.set(nodeId, newNode)
} else {
node.pattern = newNode.pattern
if (node.outerExprId !== newNode.outerExprId) {
node.outerExprId = newNode.outerExprId
}
node.rootSpan = newNode.rootSpan
}
if (nodeMeta) {
this.assignUpdatedMetadata(node ?? newNode, nodeMeta)
const differentOrDirty = (a: Ast.Ast | undefined, b: Ast.Ast | undefined) =>
a?.id !== b?.id || (a && subtreeDirty(a.id))
if (differentOrDirty(node.pattern, newNode.pattern)) node.pattern = newNode.pattern
if (node.outerExprId !== newNode.outerExprId) node.outerExprId = newNode.outerExprId
if (differentOrDirty(node.rootSpan, newNode.rootSpan)) node.rootSpan = newNode.rootSpan
}
}
for (const nodeId of this.nodeIdToNode.keys()) {
if (!currentNodeIds.has(nodeId)) {
this.nodeIdToNode.delete(nodeId)
@ -377,13 +384,15 @@ export class GraphDb {
updateMap(this.idFromExternalMap, idFromExternalNew)
}
assignUpdatedMetadata(node: Node, meta: NodeMetadata) {
const newPosition = new Vec2(meta.x, -meta.y)
if (!node.position.equals(newPosition)) {
node.position = newPosition
}
if (!visMetadataEquals(node.vis, meta.vis)) {
node.vis = meta.vis
updateMetadata(id: Ast.AstId, changes: NodeMetadata) {
const node = this.nodeIdToNode.get(id as NodeId)
if (!node) return
const newPos = changes.get('position')
const newPosVec = newPos && new Vec2(newPos.x, newPos.y)
if (newPosVec && !newPosVec.equals(node.position)) node.position = newPosVec
if (changes.has('visualization')) {
const newVis = changes.get('visualization')
if (!visMetadataEquals(newVis, node.vis)) node.vis = newVis
}
}

3
app/gui2/src/stores/graph/imports.ts
View File

@ -329,6 +329,9 @@ export function filterOutRedundantImports(
if (import.meta.vitest) {
const { test, expect } = import.meta.vitest
const { initializeFFI } = await import('shared/ast/ffi')
await initializeFFI()
test.each([
{

344
app/gui2/src/stores/graph/index.ts
View File

@ -1,7 +1,7 @@
import { nonDictatedPlacement } from '@/components/ComponentBrowser/placement'
import type { PortId } from '@/providers/portInfo'
import type { WidgetUpdate } from '@/providers/widgetRegistry'
import { asNodeId, GraphDb, type NodeId } from '@/stores/graph/graphDatabase'
import { GraphDb, type NodeId } from '@/stores/graph/graphDatabase'
import {
addImports,
filterOutRedundantImports,
@ -11,15 +11,9 @@ import {
import { useProjectStore } from '@/stores/project'
import { useSuggestionDbStore } from '@/stores/suggestionDatabase'
import { assert, bail } from '@/util/assert'
import { Ast, RawAst } from '@/util/ast'
import {
isIdentifier,
MutableModule,
ReactiveModule,
type AstId,
type Module,
} from '@/util/ast/abstract'
import { useObserveYjs } from '@/util/crdt'
import { Ast, RawAst, visitRecursive } from '@/util/ast'
import type { AstId, Module, NodeMetadata, NodeMetadataFields } from '@/util/ast/abstract'
import { MutableModule, ReactiveModule, isIdentifier } from '@/util/ast/abstract'
import { partition } from '@/util/data/array'
import type { Opt } from '@/util/data/opt'
import { Rect } from '@/util/data/rect'
@ -28,21 +22,18 @@ import { map, set } from 'lib0'
import { defineStore } from 'pinia'
import type { ExpressionUpdate, StackItem } from 'shared/languageServerTypes'
import {
IdMap,
sourceRangeKey,
visMetadataEquals,
type ExternalId,
type NodeMetadata,
type SourceRange,
type SourceRangeKey,
type VisualizationIdentifier,
type VisualizationMetadata,
} from 'shared/yjsModel'
import { computed, markRaw, reactive, ref, toRef, watch, type Ref, type ShallowRef } from 'vue'
import { computed, markRaw, reactive, ref, toRef, watch, type ShallowRef } from 'vue'
import * as Y from 'yjs'
export { type Node, type NodeId } from '@/stores/graph/graphDatabase'
const DEBUG = false
export interface NodeEditInfo {
id: NodeId
initialCursorPos: number
@ -62,39 +53,11 @@ export const useGraphStore = defineStore('graph', () => {
proj.setObservedFileName('Main.enso')
const data = computed(() => proj.module?.doc.data)
const metadata = computed(() => proj.module?.doc.metadata)
const syncModule = computed(() => proj.module && new MutableModule(proj.module.doc.ydoc))
const moduleCode = ref(proj.module?.doc.getCode())
// We need casting here, as type changes in Ref when class has private fields.
// see https://github.com/vuejs/core/issues/2557
const idMap = ref(proj.module?.doc.getIdMap()) as Ref<IdMap | undefined>
const syncModule = MutableModule.Transient()
const astModule = new ReactiveModule(syncModule)
const moduleRoot = computed(() => astModule.root()?.id)
let moduleDirty = false
const nodeRects = reactive(new Map<NodeId, Rect>())
const vizRects = reactive(new Map<NodeId, Rect>())
const topLevel = computed(() => {
// The top level of the module is always a block.
const root = moduleRoot.value
const topLevel = root != null ? astModule.get(root) : null
if (topLevel != null && !(topLevel instanceof Ast.BodyBlock)) {
return null
} else {
return topLevel
}
})
// Initialize text and idmap once module is loaded (data != null)
watch(data, () => {
if (!moduleCode.value) {
moduleCode.value = proj.module?.doc.getCode()
idMap.value = proj.module?.doc.getIdMap()
}
})
const db = new GraphDb(
suggestionDb.entries,
toRef(suggestionDb, 'groups'),
@ -107,82 +70,91 @@ export const useGraphStore = defineStore('graph', () => {
const unconnectedEdge = ref<UnconnectedEdge>()
useObserveYjs(data, (event) => {
moduleDirty = false
if (!event.changes.keys.size) return
const code = proj.module?.doc.getCode()
if (code) moduleCode.value = code
const ids = proj.module?.doc.getIdMap()
if (ids) idMap.value = ids
const moduleCode = ref<string>()
const moduleData = ref<{
getSpan: (id: Ast.AstId) => SourceRange | undefined
toRaw: (id: Ast.AstId) => RawAst.Tree.Tree | undefined
}>()
const astModule = ref<Module>(new Ast.EmptyModule())
const moduleRoot = ref<Ast.Ast>()
const topLevel = ref<Ast.BodyBlock>()
watch(syncModule, (syncModule) => {
if (!syncModule) return
const _astModule = new ReactiveModule(syncModule, [handleModuleUpdate])
})
let prevCode: string | undefined = undefined
const moduleData: {
getSpan?: (id: Ast.AstId) => SourceRange | undefined
toRaw?: Map<Ast.AstId, RawAst.Tree.Tree>
} = {}
watch([moduleCode, idMap], ([code, ids]) => {
if (!code || !ids) return
if (DEBUG && code === prevCode) console.info(`moduleData: Code is unchanged.`)
const edit = syncModule.edit()
const {
root,
idMap: parsedIds,
getSpan,
toRaw,
idMapUpdates,
} = Ast.parseExtended(code, ids, edit)
Y.applyUpdateV2(syncModule.ydoc, Y.encodeStateAsUpdateV2(root.module.ydoc))
db.updateExternalIds(root)
moduleData.toRaw = toRaw
moduleData.getSpan = getSpan
prevCode = code
proj.module!.transact(() => {
if (idMapUpdates) proj.module!.doc.setIdMap(parsedIds)
updateState()
})
})
function handleModuleUpdate(
module: Module,
dirtyNodes: Iterable<AstId>,
metadataUpdates: { id: AstId; changes: Map<string, unknown> }[],
) {
const moduleChanged = astModule.value !== reactive(module)
if (moduleChanged) {
if (astModule.value instanceof ReactiveModule) astModule.value.disconnect()
astModule.value = module
}
const root = module.root()
if (!root) return
moduleRoot.value = root
if (root instanceof Ast.BodyBlock) topLevel.value = root
// We can cast maps of unknown metadata fields to `NodeMetadata` because all `NodeMetadata` fields are optional.
const nodeMetadataUpdates = metadataUpdates as any as { id: AstId; changes: NodeMetadata }[]
const dirtyNodeSet = new Set(dirtyNodes)
if (moduleChanged || dirtyNodeSet.size !== 0) {
const { code, info } = Ast.print(root)
moduleCode.value = code
db.updateExternalIds(root)
const getSpan = Ast.spanMapToSpanGetter(info)
const toRawMap = new Map<SourceRangeKey, RawAst.Tree>()
visitRecursive(Ast.parseEnso(code), (node) => {
if (node.type === RawAst.Tree.Type.Function) {
const start = node.whitespaceStartInCodeParsed + node.whitespaceLengthInCodeParsed
const end = start + node.childrenLengthInCodeParsed
toRawMap.set(sourceRangeKey([start, end]), node)
}
return true
})
const toRaw = (id: Ast.AstId) => {
const span = getSpan(id)
if (!span) return
return toRawMap.get(sourceRangeKey(span))
}
moduleData.value = { toRaw, getSpan }
updateState(dirtyNodeSet)
}
if (nodeMetadataUpdates.length !== 0) {
for (const { id, changes } of nodeMetadataUpdates) db.updateMetadata(id, changes)
}
}
function updateState() {
if (!moduleData.toRaw || !moduleData.getSpan) return
function updateState(dirtyNodes?: Set<AstId>) {
if (!moduleData.value) return
const module = proj.module
if (!module) return
const meta = module.doc.metadata
const textContentLocal = moduleCode.value
if (!textContentLocal) return
methodAst.value = methodAstInModule(astModule)
methodAst.value = methodAstInModule(astModule.value)
if (methodAst.value) {
const rawFunc = moduleData.toRaw.get(methodAst.value.id)
const rawFunc = moduleData.value.toRaw(methodAst.value.id)
assert(rawFunc?.type === RawAst.Tree.Type.Function)
currentNodeIds.value = db.readFunctionAst(
methodAst.value,
rawFunc,
textContentLocal,
(id) => meta.get(id),
moduleData.getSpan,
moduleData.value.getSpan,
dirtyNodes ?? new Set(),
)
}
}
function methodAstInModule(mod: Module) {
const topLevel = mod.root()
if (!topLevel) return
assert(topLevel instanceof Ast.BodyBlock)
return getExecutedMethodAst(topLevel, proj.executionContext.getStackTop(), db)
}
useObserveYjs(metadata, (event) => {
const meta = event.target
for (const [key, op] of event.changes.keys) {
if (op.action === 'update' || op.action === 'add') {
const externalId = key as ExternalId
const data = meta.get(externalId)
const id = db.idFromExternal(externalId)
const node = id && db.nodeIdToNode.get(asNodeId(id))
if (data && node) db.assignUpdatedMetadata(node, data)
}
}
})
function generateUniqueIdent() {
for (;;) {
const ident = randomIdent()
@ -229,20 +201,13 @@ export const useGraphStore = defineStore('graph', () => {
function createNode(
position: Vec2,
expression: string,
metadata: NodeMetadata | undefined = undefined,
metadata: NodeMetadataFields = {},
withImports: RequiredImport[] | undefined = undefined,
): Opt<NodeId> {
const mod = proj.module
if (!mod) return
const meta = metadata ?? {
x: position.x,
y: -position.y,
vis: null,
}
meta.x = position.x
meta.y = -position.y
const ident = generateUniqueIdent()
const edit = astModule.edit()
const edit = astModule.value.edit()
if (withImports) addMissingImports(edit, withImports)
const currentFunc = 'main'
const method = Ast.findModuleMethod(topLevel.value!, currentFunc)
@ -252,14 +217,16 @@ export const useGraphStore = defineStore('graph', () => {
}
const functionBlock = edit.getVersion(method).bodyAsBlock()
const rhs = Ast.parse(expression, edit)
metadata.position = { x: position.x, y: position.y }
rhs.setNodeMetadata(metadata)
const assignment = Ast.Assignment.new(edit, ident, rhs)
functionBlock.push(assignment)
commitEdit(edit, new Map([[rhs.id, meta]]))
commitEdit(edit)
}
function addMissingImports(edit: MutableModule, newImports: RequiredImport[]) {
if (!newImports.length) return
const topLevel = edit.get(moduleRoot.value)
const topLevel = edit.getVersion(moduleRoot.value!)
if (!(topLevel instanceof Ast.MutableBodyBlock)) {
console.error(`BUG: Cannot add required imports: No BodyBlock module root.`)
return
@ -271,24 +238,23 @@ export const useGraphStore = defineStore('graph', () => {
}
function deleteNodes(ids: NodeId[]) {
const edit = astModule.edit()
for (const id of ids) {
const node = db.nodeIdToNode.get(id)
if (!node) return
proj.module?.doc.metadata.delete(node.rootSpan.externalId)
const outerExpr = edit.get(node.outerExprId)
if (outerExpr) Ast.deleteFromParentBlock(outerExpr)
nodeRects.delete(id)
}
commitEdit(edit)
commitDirect((edit) => {
for (const id of ids) {
const node = db.nodeIdToNode.get(id)
if (!node) return
const outerExpr = edit.get(node.outerExprId)
if (outerExpr) Ast.deleteFromParentBlock(outerExpr)
nodeRects.delete(id)
}
}, true)
}
function setNodeContent(id: NodeId, content: string) {
const node = db.nodeIdToNode.get(id)
if (!node) return
const edit = astModule.edit()
edit.getVersion(node.rootSpan).replaceValue(Ast.parse(content, edit))
commitEdit(edit)
commitDirect((edit) => {
edit.getVersion(node.rootSpan).replaceValue(Ast.parse(content, edit))
})
}
function transact(fn: () => void) {
@ -300,56 +266,56 @@ export const useGraphStore = defineStore('graph', () => {
}
function setNodePosition(nodeId: NodeId, position: Vec2) {
const externalId = db.idToExternal(nodeId)
if (!externalId) {
if (DEBUG) console.warn(`setNodePosition: Node not found.`)
return
const nodeAst = astModule.value?.get(nodeId)
if (!nodeAst) return
const oldPos = nodeAst.nodeMetadata.get('position')
if (oldPos?.x !== position.x || oldPos?.y !== position.y) {
commitDirect((edit) => {
edit
.getVersion(nodeAst)
.mutableNodeMetadata()
.set('position', { x: position.x, y: position.y })
}, true)
}
proj.module?.updateNodeMetadata(externalId, { x: position.x, y: -position.y })
}
function normalizeVisMetadata(
id: Opt<VisualizationIdentifier>,
visible: boolean | undefined,
): VisualizationMetadata | null {
): VisualizationMetadata | undefined {
const vis: VisualizationMetadata = { identifier: id ?? null, visible: visible ?? false }
if (visMetadataEquals(vis, { identifier: null, visible: false })) return null
if (visMetadataEquals(vis, { identifier: null, visible: false })) return undefined
else return vis
}
function setNodeVisualizationId(nodeId: NodeId, vis: Opt<VisualizationIdentifier>) {
const node = db.nodeIdToNode.get(nodeId)
if (!node) return
const externalId = db.idToExternal(nodeId)
if (!externalId) {
if (DEBUG) console.warn(`setNodeVisualizationId: Node not found.`)
return
}
proj.module?.updateNodeMetadata(externalId, {
vis: normalizeVisMetadata(vis, node.vis?.visible),
})
const nodeAst = astModule.value?.get(nodeId)
if (!nodeAst) return
commitDirect((edit) => {
const metadata = edit.getVersion(nodeAst).mutableNodeMetadata()
metadata.set(
'visualization',
normalizeVisMetadata(vis, metadata.get('visualization')?.visible),
)
}, true)
}
function setNodeVisualizationVisible(nodeId: NodeId, visible: boolean) {
const node = db.nodeIdToNode.get(nodeId)
if (!node) return
const externalId = db.idToExternal(nodeId)
if (!externalId) {
if (DEBUG) console.warn(`setNodeVisualizationId: Node not found.`)
return
}
proj.module?.updateNodeMetadata(externalId, {
vis: normalizeVisMetadata(node.vis?.identifier, visible),
})
const nodeAst = astModule.value?.get(nodeId)
if (!nodeAst) return
commitDirect((edit) => {
const metadata = edit.getVersion(nodeAst).mutableNodeMetadata()
metadata.set(
'visualization',
normalizeVisMetadata(metadata.get('visualization')?.identifier, visible),
)
}, true)
}
function updateNodeRect(nodeId: NodeId, rect: Rect) {
const externalId = db.idToExternal(nodeId)
if (!externalId) {
if (DEBUG) console.warn(`updateNodeRect: Node not found.`)
return
}
if (rect.pos.equals(Vec2.Zero) && !metadata.value?.has(externalId)) {
const nodeAst = astModule.value?.get(nodeId)
if (!nodeAst) return
if (rect.pos.equals(Vec2.Zero) && !nodeAst.nodeMetadata.get('position')) {
const { position } = nonDictatedPlacement(rect.size, {
nodeRects: [...nodeRects.entries()]
.filter(([id]) => db.nodeIdToNode.get(id))
@ -359,12 +325,12 @@ export const useGraphStore = defineStore('graph', () => {
screenBounds: Rect.Zero,
mousePosition: Vec2.Zero,
})
const node = db.nodeIdToNode.get(nodeId)
metadata.value?.set(externalId, {
x: position.x,
y: -position.y,
vis: node?.vis ?? null,
})
commitDirect((edit) => {
edit
.getVersion(nodeAst)
.mutableNodeMetadata()
.set('position', { x: position.x, y: position.y })
}, true)
nodeRects.set(nodeId, new Rect(position, rect.size))
} else {
nodeRects.set(nodeId, rect)
@ -435,32 +401,40 @@ export const useGraphStore = defineStore('graph', () => {
return true
}
function commitEdit(edit: MutableModule, metadataUpdates?: Map<AstId, Partial<NodeMetadata>>) {
/** Apply the given `edit` to the state.
*
* @param skipTreeRepair - If the edit is known not to require any parenthesis insertion, this may be set to `true`
* for better performance.
*/
function commitEdit(edit: MutableModule, skipTreeRepair?: boolean) {
const root = edit.root()
if (!root) {
console.error(`BUG: Cannot commit edit: No module root.`)
if (!(root instanceof Ast.BodyBlock)) {
console.error(`BUG: Cannot commit edit: No module root block.`)
return
}
const printed = Ast.print(root)
const module_ = proj.module
if (!module_) return
if (moduleDirty) {
console.warn(
`An edit has been committed before a previous edit has been observed. The new edit will supersede the previous edit.`,
)
}
moduleDirty = true
module_.transact(() => {
const idMap = Ast.spanMapToIdMap(printed.info)
module_.doc.setIdMap(idMap)
module_.doc.setCode(printed.code)
if (DEBUG) console.info(`commitEdit`, idMap, printed.code)
if (metadataUpdates) {
for (const [id, meta] of metadataUpdates) {
module_.updateNodeMetadata(edit.checkedGet(id).externalId, meta)
}
if (!skipTreeRepair) Ast.repair(root, edit)
Y.applyUpdateV2(syncModule.value!.ydoc, Y.encodeStateAsUpdateV2(edit.ydoc), 'local')
}
/** Run the given callback with direct access to the document module. Any edits to the module will be committed
* unconditionally; use with caution to avoid committing partial edits.
*
* @param skipTreeRepair - If the edit is known not to require any parenthesis insertion, this may be set to `true`
* for better performance.
*/
function commitDirect(f: (edit: MutableModule) => void, skipTreeRepair?: boolean) {
const edit = syncModule.value
assert(edit != null)
edit.ydoc.transact(() => {
f(edit)
if (!skipTreeRepair) {
const root = edit.root()
assert(root instanceof Ast.BodyBlock)
Ast.repair(root, edit)
}
})
}, 'local')
}
function mockExpressionUpdate(
@ -496,12 +470,6 @@ export const useGraphStore = defineStore('graph', () => {
proj.computedValueRegistry.processUpdates([update_])
}
function editScope(scope: (edit: MutableModule) => Map<AstId, Partial<NodeMetadata>> | void) {
const edit = astModule.edit()
const metadataUpdates = scope(edit)
commitEdit(edit, metadataUpdates ?? undefined)
}
/**
* Reorders nodes so the `targetNodeId` node is placed after `sourceNodeId`. Does nothing if the
* relative order is already correct.
@ -593,7 +561,7 @@ export const useGraphStore = defineStore('graph', () => {
setEditedNode,
updateState,
commitEdit,
editScope,
commitDirect,
addMissingImports,
}
})

3
app/gui2/src/stores/suggestionDatabase/__tests__/lsUpdate.test.ts
View File

@ -4,9 +4,12 @@ import { applyUpdates } from '@/stores/suggestionDatabase/lsUpdate'
import { unwrap } from '@/util/data/result'
import { parseDocs } from '@/util/docParser'
import { tryIdentifier, tryQualifiedName, type QualifiedName } from '@/util/qualifiedName'
import { initializeFFI } from 'shared/ast/ffi'
import * as lsTypes from 'shared/languageServerTypes/suggestions'
import { expect, test } from 'vitest'
await initializeFFI()
test('Adding suggestion database entries', () => {
const test = new Fixture()
const db = new SuggestionDb()

2
app/gui2/src/stores/suggestionDatabase/documentation.ts
View File

@ -65,6 +65,8 @@ export function documentationData(
if (import.meta.vitest) {
const { test, expect } = import.meta.vitest
const { initializeFFI } = await import('shared/ast/ffi')
await initializeFFI()
test.each([
['ALIAS Bar', 'Bar'],

70
app/gui2/src/util/assert.ts
View File

@ -1,69 +1 @@
export function assertNever(x: never): never {
bail('Unexpected object: ' + JSON.stringify(x))
}
export function assert(condition: boolean, message?: string): asserts condition {
if (!condition) bail(message ? `Assertion failed: ${message}` : 'Assertion failed')
}
/**
* Checks if the given iterable has the specified length and throws an assertion error
* if the lengths do not match.
*
* @param iterable The iterable to check.
* @param length The expected length of the iterable.
* @param message Optional message for the assertion error.
* @return void
* @throws Error Will throw an error if the length does not match.
*
* The first five elements of the iterable will be displayed in the error message
* if the assertion fails. If the iterable contains more than five elements,
* the remaining elements will be represented as '...'.
*/
export function assertLength<T>(iterable: Iterable<T>, length: number, message?: string): void {
const convertedArray = Array.from(iterable)
const messagePrefix = message ? message + ' ' : ''
const elementRepresentation =
convertedArray.length > 5
? `${convertedArray.slice(0, 5).join(', ')},...`
: convertedArray.join(', ')
assert(
convertedArray.length === length,
`${messagePrefix}Expected iterable of length ${length}, got length ${convertedArray.length}. Elements: [${elementRepresentation}]`,
)
}
export function assertEmpty<T>(iterable: Iterable<T>, message?: string): void {
assertLength(iterable, 0, message)
}
export function assertEqual<T>(actual: T, expected: T, message?: string) {
const messagePrefix = message ? message + ' ' : ''
assert(actual === expected, `${messagePrefix}Expected ${expected}, got ${actual}.`)
}
export function assertNotEqual<T>(actual: T, unexpected: T, message?: string) {
const messagePrefix = message ? message + ' ' : ''
assert(actual !== unexpected, `${messagePrefix}Expected not ${unexpected}, got ${actual}.`)
}
export function assertDefined<T>(x: T | undefined, message?: string): asserts x is T {
const messagePrefix = message ? message + ' ' : ''
assert(x !== undefined, `${messagePrefix}Expected value to be defined.`)
}
export function assertUnreachable(): never {
bail('Unreachable code')
}
/**
* Throw an error with provided message.
*
* It is convenient to use at the end of a nullable chain:
* ```ts
* const x = foo?.bar.baz?.() ?? bail('Expected foo.bar.baz to exist')
* ```
*/
export function bail(message: string): never {
throw new Error(message)
}
export * from 'shared/util/assert'

75
app/gui2/src/util/ast/__tests__/abstract.test.ts
View File

@ -1,8 +1,11 @@
import { assert } from '@/util/assert'
import { Ast } from '@/util/ast'
import { initializeFFI } from 'shared/ast/ffi'
import { expect, test } from 'vitest'
import { MutableModule, escape, unescape, type Identifier } from '../abstract'
await initializeFFI()
//const disabledCases = [
// ' a',
// 'a ',
@ -356,6 +359,9 @@ test.each(cases)('parse/print round trip: %s', (code) => {
const idMap = Ast.spanMapToIdMap(printed.info)
idMap.validate()
// Parsed tree shouldn't need any repair.
expect(Ast.repair(root).fixes).toBe(undefined)
// Re-parse.
const { root: root1, spans: spans1 } = Ast.parseBlockWithSpans(printed.code)
Ast.setExternalIds(root1.module, spans1, idMap)
@ -474,6 +480,24 @@ test('Splice', () => {
expect(spliced.code()).toBe('foo')
})
test('Construct app', () => {
const edit = MutableModule.Transient()
const app = Ast.App.new(
edit,
Ast.Ident.new(edit, 'func' as Identifier),
undefined,
Ast.Ident.new(edit, 'arg' as Identifier),
)
expect(app.code()).toBe('func arg')
const namedApp = Ast.App.new(
edit,
Ast.Ident.new(edit, 'func' as Identifier),
'argName' as Identifier,
Ast.Ident.new(edit, 'arg' as Identifier),
)
expect(namedApp.code()).toBe('func argName=arg')
})
test.each([
['Hello, World!', 'Hello, World!'],
['Hello\t\tWorld!', 'Hello\\t\\tWorld!'],
@ -490,3 +514,54 @@ test.each([
expect(escaped).toBe(expectedEscaped)
expect(unescape(escaped)).toBe(original)
})
test('Automatic parenthesis', () => {
const block = Ast.parseBlock('main = func arg1 arg2')
let arg1: Ast.MutableAst | undefined
block.visitRecursiveAst((ast) => {
if (ast instanceof Ast.MutableIdent && ast.code() === 'arg1') {
assert(!arg1)
arg1 = ast
}
})
assert(arg1 != null)
arg1.replace(Ast.parse('innerfunc innerarg', block.module))
const correctCode = 'main = func (innerfunc innerarg) arg2'
// This assertion will fail when smart printing handles this case.
// At that point we should test tree repair separately.
assert(block.code() !== correctCode)
Ast.repair(block, block.module)
expect(block.code()).toBe(correctCode)
})
test('Resync', () => {
const root = Ast.parseBlock('main = func arg1 arg2')
const module = root.module
module.replaceRoot(root)
const arg1 = 'arg1' as Identifier
const func = Ast.Function.fromStatements(
module,
'func' as Identifier,
[arg1],
[Ast.Ident.new(module, arg1)],
)
// Add a trailing line to the function's block. This is syntactically non-canonical; it should belong to the parent.
func.bodyAsBlock().insert(1, undefined)
expect(func.bodyAsBlock().lines.length).toBe(2)
root.insert(0, func)
const codeBeforeRepair = root.code()
const rootExternalIdBeforeRepair = root.externalId
const funcExternalIdBeforeRepair = func.externalId
Ast.repair(root, module)
const repairedRoot = module.root()
assert(repairedRoot instanceof Ast.BodyBlock)
const repairedFunc = repairedRoot.statements().next().value
assert(repairedFunc instanceof Ast.Function)
assert(repairedFunc.body instanceof Ast.BodyBlock)
// The function's body has been corrected.
expect(repairedFunc.body.lines.length).toBe(1)
expect(repairedRoot.code()).toBe(codeBeforeRepair)
expect(repairedRoot.externalId).toBe(rootExternalIdBeforeRepair)
// The resync operation loses metadata within the non-canonical subtree.
expect(repairedFunc.body?.externalId).not.toBe(funcExternalIdBeforeRepair)
})

6
app/gui2/src/util/ast/__tests__/abstractFileIo.test.ts
View File

@ -1,5 +1,6 @@
import { Ast } from '@/util/ast'
import * as fs from 'fs'
import { initializeFFI } from 'shared/ast/ffi'
import { splitFileContents } from 'shared/ensoFile'
import { expect, test } from 'vitest'
@ -7,6 +8,8 @@ import { expect, test } from 'vitest'
// file format handling to shared and create a test utility for easy *.enso file fixture loading.
import { deserializeIdMap } from '../../../../ydoc-server/serialization'
await initializeFFI()
test('full file IdMap round trip', () => {
const content = fs.readFileSync(__dirname + '/fixtures/stargazers.enso').toString()
const { code, idMapJson, metadataJson: _ } = splitFileContents(content)
@ -20,4 +23,7 @@ test('full file IdMap round trip', () => {
expect(Ast.tokenTreeWithIds(ast2), 'Print/parse preserves IDs').toStrictEqual(astTT)
expect(Ast.tokenTreeWithIds(ast_), 'All node IDs come from IdMap').toStrictEqual(astTT)
expect([...idMap.entries()].sort()).toStrictEqual([...idMapOriginal.entries()].sort())
// Parsed tree shouldn't need any repair.
expect(Ast.repair(ast).fixes).toBe(undefined)
})

7
app/gui2/src/util/ast/__tests__/ast.test.ts
View File

@ -1,4 +1,3 @@
import { Token, Tree } from '@/generated/ast'
import {
astContainingChar,
childrenAstNodes,
@ -10,9 +9,13 @@ import {
readTokenSpan,
walkRecursive,
} from '@/util/ast'
import type { LazyObject } from '@/util/parserSupport'
import { initializeFFI } from 'shared/ast/ffi'
import { Token, Tree } from 'shared/ast/generated/ast'
import type { LazyObject } from 'shared/ast/parserSupport'
import { assert, expect, test } from 'vitest'
await initializeFFI()
function validateSpans(obj: LazyObject, initialPos?: number): number {
const state = { pos: initialPos ?? 0 }
const visitor = (value: LazyObject) => {

3
app/gui2/src/util/ast/__tests__/callTree.test.ts
View File

@ -7,8 +7,11 @@ import {
ArgumentPlaceholder,
interpretCall,
} from '@/util/callTree'
import { initializeFFI } from 'shared/ast/ffi'
import { assert, expect, test } from 'vitest'
await initializeFFI()
const prefixFixture = {
allowInfix: false,
mockSuggestion: {

3
app/gui2/src/util/ast/__tests__/match.test.ts
View File

@ -1,8 +1,11 @@
import { Ast } from '@/util/ast'
import { Pattern } from '@/util/ast/match'
import { initializeFFI } from 'shared/ast/ffi'
import { expect, test } from 'vitest'
import { MutableModule } from '../abstract'
await initializeFFI()
test.each([
{ target: 'a.b', pattern: '__', extracted: ['a.b'] },
{ target: 'a.b', pattern: 'a.__', extracted: ['b'] },

3
app/gui2/src/util/ast/__tests__/prefixes.test.ts
View File

@ -1,7 +1,10 @@
import { Ast } from '@/util/ast/abstract'
import { Prefixes } from '@/util/ast/prefixes'
import { initializeFFI } from 'shared/ast/ffi'
import { expect, test } from 'vitest'
await initializeFFI()
test.each([
{
prefixes: {

24
app/gui2/src/util/ast/__tests__/reactive.test.ts
View File

@ -1,17 +1,37 @@
import { Ast } from '@/util/ast'
import { MutableModule, ReactiveModule } from '@/util/ast/abstract'
import { MutableModule, newExternalId, ReactiveModule } from '@/util/ast/abstract'
import { initializeFFI } from 'shared/ast/ffi'
import { expect, test } from 'vitest'
import * as Y from 'yjs'
test('Reactive module observes Y.Js changes', () => {
await initializeFFI()
test('Reactive module observes tree changes', () => {
const syncModule = MutableModule.Transient()
const reactiveModule = new ReactiveModule(syncModule)
const { root } = Ast.parseExtended('main =\n 23')
Y.applyUpdateV2(syncModule.ydoc, Y.encodeStateAsUpdateV2(root.module.ydoc))
expect(reactiveModule.root()?.code()).toBe('main =\n 23')
expect(reactiveModule.root()?.externalId).toBe(root.externalId)
const edit = syncModule.edit()
const { root: root2 } = Ast.parseExtended('main = 42', undefined, edit)
Y.applyUpdateV2(syncModule.ydoc, Y.encodeStateAsUpdateV2(root2.module.ydoc))
expect(reactiveModule.root()?.code()).toBe('main = 42')
expect(reactiveModule.root()?.externalId).toBe(root2.externalId)
})
test('Reactive module observes metadata changes', () => {
const syncModule = MutableModule.Transient()
const reactiveModule = new ReactiveModule(syncModule)
const { root } = Ast.parseExtended('main =\n 23')
Y.applyUpdateV2(syncModule.ydoc, Y.encodeStateAsUpdateV2(root.module.ydoc))
expect(reactiveModule.root()?.code()).toBe('main =\n 23')
expect(reactiveModule.root()?.externalId).toBe(root.externalId)
const edit = syncModule.edit()
const newId = newExternalId()
edit.getVersion(root).setExternalId(newId)
Y.applyUpdateV2(syncModule.ydoc, Y.encodeStateAsUpdateV2(edit.ydoc))
expect(reactiveModule.root()?.externalId).toBe(newId)
})

2974
app/gui2/src/util/ast/abstract.ts
View File

File diff suppressed because it is too large Load Diff

5
app/gui2/src/util/ast/aliasAnalysis.ts
View File

@ -8,9 +8,12 @@ import {
readTokenSpan,
} from '@/util/ast'
import { MappedKeyMap, MappedSet, NonEmptyStack } from '@/util/containers'
import type { LazyObject } from '@/util/parserSupport'
import { initializeFFI } from 'shared/ast/ffi'
import type { LazyObject } from 'shared/ast/parserSupport'
import { rangeIsBefore, sourceRangeKey, type SourceRange } from 'shared/yjsModel'
await initializeFFI()
const ACCESSOR_OPERATOR = '.'
const LAMBDA_OPERATOR = '->'

4
app/gui2/src/util/ast/extended.ts
View File

@ -1,5 +1,3 @@
import * as Ast from '@/generated/ast'
import { Token, Tree } from '@/generated/ast'
import { assert } from '@/util/assert'
import {
childrenAstNodesOrTokens,
@ -14,6 +12,8 @@ import type { Opt } from '@/util/data/opt'
import * as encoding from 'lib0/encoding'
import * as sha256 from 'lib0/hash/sha256'
import * as map from 'lib0/map'
import * as Ast from 'shared/ast/generated/ast'
import { Token, Tree } from 'shared/ast/generated/ast'
import type { ExternalId, IdMap, SourceRange } from 'shared/yjsModel'
import { markRaw } from 'vue'

15
app/gui2/src/util/ast/index.ts
View File

@ -1,28 +1,23 @@
import * as RawAst from '@/generated/ast'
import { assert } from '@/util/assert'
import * as Ast from '@/util/ast/abstract'
import { parseEnso } from '@/util/ast/abstract'
import { AstExtended as RawAstExtended } from '@/util/ast/extended'
import { isResult, mapOk } from '@/util/data/result'
import { parse } from '@/util/ffi'
import { LazyObject, LazySequence } from '@/util/parserSupport'
import * as map from 'lib0/map'
import * as RawAst from 'shared/ast/generated/ast'
import { LazyObject, LazySequence } from 'shared/ast/parserSupport'
import type { SourceRange } from 'shared/yjsModel'
export { Ast, RawAst, RawAstExtended }
export { Ast, RawAst, RawAstExtended, parseEnso }
export type HasAstRange = SourceRange | RawAst.Tree | RawAst.Token
export function parseEnso(code: string): RawAst.Tree {
const blob = parse(code)
return RawAst.Tree.read(new DataView(blob.buffer), blob.byteLength - 4)
}
/** Read a single line of code
*
* Is meant to be a helper for tests. If the code is multiline, an exception is raised.
*/
export function parseEnsoLine(code: string): RawAst.Tree {
const block = parseEnso(code)
const block = Ast.parseEnso(code)
assert(block.type === RawAst.Tree.Type.BodyBlock)
const statements = block.statements[Symbol.iterator]()
const firstLine = statements.next()

3
app/gui2/src/util/ast/opr.ts
View File

@ -155,6 +155,9 @@ export function* operandsOfLeftAssocOprChain<HasIdMap extends boolean = true>(
if (import.meta.vitest) {
const { test, expect } = import.meta.vitest
const { initializeFFI } = await import('shared/ast/ffi')
await initializeFFI()
test.each([
{ code: '2 + 3', result: ['2', '+', '3'] },

7
app/gui2/src/util/config.ts
View File

@ -143,9 +143,12 @@ export type ConfigValue<T extends Config<any> = Config> = ConfigOptionValues<T>
ConfigGroupValues<T>
export function configValue<T extends Config<any>>(config: T): ConfigValue<T> {
// The object may be undefined if we pass an incomplete `ApplicationConfigValue` to `provideGuiConfig._mock`.
const options = config.options ?? {}
const groups = config.groups ?? {}
return Object.fromEntries([
...Object.entries(config.options).map(([k, v]) => [k, optionValue(v as Option)]),
...Object.entries(config.groups).map(([k, v]) => [k, groupValue(v as Group)]),
...Object.entries(options).map(([k, v]) => [k, optionValue(v as Option)]),
...Object.entries(groups).map(([k, v]) => [k, groupValue(v as Group)]),
]) satisfies ConfigValue as any
}

26
app/gui2/src/util/data/opt.ts
View File

@ -1,25 +1 @@
/** @file A value that may be `null` or `undefined`. */
/** Optional value type. This is a replacement for `T | null | undefined` that is more
* convenient to use. We do not select a single value to represent "no value", because we are using
* libraries that disagree whether `null` (e.g. Yjs) or `undefined` (e.g. Vue) should be used for
* that purpose. We want to be compatible with both without needless conversions. In our own code,
* we should return `undefined` for "no value", since that is the default value for empty or no
* `return` expression. In order to test whether an `Opt<T>` is defined or not, use `x == null` or
* `isSome` function.
*
* Note: For JSON-serialized data, prefer explicit `null` over `undefined`, since `undefined` is
* not serializable. Alternatively, use optional field syntax (e.g. `{ x?: number }`). */
export type Opt<T> = T | null | undefined
export function isSome<T>(value: Opt<T>): value is T {
return value != null
}
export function isNone(value: Opt<any>): value is null | undefined {
return value == null
}
export function mapOr<T, R>(optional: Opt<T>, fallback: R, mapper: (value: T) => R): R {
return isSome(optional) ? mapper(optional) : fallback
}
export * from 'shared/util/data/opt'

86
app/gui2/src/util/data/result.ts
View File

@ -1,85 +1 @@
/** @file A generic type that can either hold a value representing a successful result,
* or an error. */
import { isSome, type Opt } from '@/util/data/opt'
export type Result<T = undefined, E = string> =
| { ok: true; value: T }
| { ok: false; error: ResultError<E> }
export function Ok<T>(data: T): Result<T, never> {
return { ok: true, value: data }
}
export function Err<E>(error: E): Result<never, E> {
return { ok: false, error: new ResultError(error) }
}
export function okOr<T, E>(data: Opt<T>, error: E): Result<T, E> {
if (isSome(data)) return Ok(data)
else return Err(error)
}
export function unwrap<T, E>(result: Result<T, E>): T {
if (result.ok) return result.value
else throw result.error
}
export function mapOk<T, U, E>(result: Result<T, E>, f: (value: T) => U): Result<U, E> {
if (result.ok) return Ok(f(result.value))
else return result
}
export function isResult(v: unknown): v is Result {
return (
v != null &&
typeof v === 'object' &&
'ok' in v &&
typeof v.ok === 'boolean' &&
('value' in v || ('error' in v && v.error instanceof ResultError))
)
}
export class ResultError<E = string> {
payload: E
context: (() => string)[]
constructor(payload: E) {
this.payload = payload
this.context = []
}
log(preamble: string = 'Error') {
const ctx =
this.context.length > 0 ? `\n${Array.from(this.context, (ctx) => ctx()).join('\n')}` : ''
console.error(`${preamble}: ${this.payload}${ctx}`)
}
}
export function withContext<T, E>(context: () => string, f: () => Result<T, E>): Result<T, E> {
const result = f()
if (result == null) {
throw new Error('withContext: f() returned null or undefined')
}
if (!result.ok) result.error.context.splice(0, 0, context)
return result
}
/**
* Catch promise rejection of provided types and convert them to a Result type.
*/
export function rejectionToResult<ErrorKind extends new (...args: any[]) => any>(
errorKinds: ErrorKind | ErrorKind[],
): <T>(promise: Promise<T>) => Promise<Result<T, InstanceType<ErrorKind>>> {
const errorKindArray = Array.isArray(errorKinds) ? errorKinds : [errorKinds]
return async (promise) => {
try {
return Ok(await promise)
} catch (error) {
for (const errorKind of errorKindArray) {
if (error instanceof errorKind) return Err(error)
}
throw error
}
}
}
export * from 'shared/util/data/result'

2
app/gui2/src/util/docParser.ts
View File

@ -1,4 +1,4 @@
import { parse_doc_to_json } from '@/util/ffi'
import { parse_doc_to_json } from 'shared/ast/ffi'
export function parseDocs(docs: string): Doc.Section[] {
const json = parse_doc_to_json(docs)

13
app/gui2/src/util/ffi.ts
View File

@ -1,13 +0,0 @@
import { isNode } from '@/util/detect'
import init, { is_ident_or_operator, parse, parse_doc_to_json } from 'rust-ffi/pkg/rust_ffi'
if (isNode) {
const fs = await import('node:fs/promises')
const buffer = await fs.readFile('./rust-ffi/pkg/rust_ffi_bg.wasm')
await init(buffer)
} else {
await init()
}
// eslint-disable-next-line camelcase
export { is_ident_or_operator, parse, parse_doc_to_json }

2
app/gui2/src/util/qualifiedName.ts
View File

@ -100,6 +100,8 @@ export function qnIsTopElement(name: QualifiedName): boolean {
if (import.meta.vitest) {
const { test, expect } = import.meta.vitest
const { initializeFFI } = await import('shared/ast/ffi')
await initializeFFI()
const validIdentifiers = [
'A',

54
app/gui2/stories/MockProjectStoreWrapper.vue
View File

@ -1,7 +1,7 @@
<script setup lang="ts">
import { useProjectStore } from '@/stores/project'
import { useObserveYjs } from '@/util/crdt'
import { computed, watchEffect } from 'vue'
import { Ast } from '@/util/ast'
import { watch } from 'vue'
const props = defineProps<{ modelValue: string }>()
const emit = defineEmits<{ 'update:modelValue': [modelValue: string] }>()
@ -10,26 +10,40 @@ const projectStore = useProjectStore()
const mod = projectStore.projectModel.createNewModule('Main.enso')
projectStore.setObservedFileName('Main.enso')
mod.doc.ydoc.emit('load', [])
let syncedCode: string | undefined
watch(
() => projectStore.module,
(mod) => {
if (!mod) return
const syncModule = new Ast.MutableModule(mod.doc.ydoc)
applyEdits(syncModule, props.modelValue)
const _astModule = new Ast.ReactiveModule(syncModule, [
(module, dirtyNodes) => {
if (dirtyNodes.size === 0) return
const root = module.root()
if (root) {
const { code } = Ast.print(root)
if (code !== props.modelValue) {
syncedCode = code
emit('update:modelValue', code)
}
}
},
])
watch(
() => props.modelValue,
(modelValue) => applyEdits(syncModule, modelValue),
)
},
)
function applyEdits(module: NonNullable<typeof projectStore.module>, newText: string) {
module.transact(() => {
projectStore.module?.doc.setCode(newText)
})
}
watchEffect(() => projectStore.module && applyEdits(projectStore.module, props.modelValue))
const data = computed(() => projectStore.module?.doc.data)
const text = computed(() => projectStore.module?.doc.getCode())
useObserveYjs(data, () => {
if (text.value) {
const newValue = text.value?.toString()
if (newValue !== props.modelValue) {
emit('update:modelValue', newValue)
}
function applyEdits(syncModule: Ast.MutableModule, newText: string) {
if (newText !== syncedCode) {
syncModule.ydoc.transact(() => {
syncModule.syncRoot(Ast.parseBlock(newText, syncModule))
}, 'local')
}
})
}
</script>
<template>

1
app/gui2/vite.config.ts
View File

@ -35,7 +35,6 @@ export default defineConfig({
? { '/src/main.ts': fileURLToPath(new URL('./e2e/main.ts', import.meta.url)) }
: {}),
shared: fileURLToPath(new URL('./shared', import.meta.url)),
'rust-ffi': fileURLToPath(new URL('./rust-ffi', import.meta.url)),
'@': fileURLToPath(new URL('./src', import.meta.url)),
},
},

136
app/gui2/ydoc-server/edits.ts
View File

@ -4,117 +4,69 @@
*/
import diff from 'fast-diff'
import * as json from 'lib0/json'
import * as Y from 'yjs'
import { combineFileParts, splitFileContents } from '../shared/ensoFile'
import type { ModuleUpdate } from '../shared/ast'
import { MutableModule, print, spanMapToIdMap } from '../shared/ast'
import { EnsoFileParts } from '../shared/ensoFile'
import { TextEdit } from '../shared/languageServerTypes'
import { ModuleDoc, type NodeMetadata, type VisualizationMetadata } from '../shared/yjsModel'
import { assert } from '../shared/util/assert'
import { IdMap, ModuleDoc, type VisualizationMetadata } from '../shared/yjsModel'
import * as fileFormat from './fileFormat'
import { serializeIdMap } from './serialization'
interface AppliedUpdates {
edits: TextEdit[]
newContent: string
newMetadata: fileFormat.Metadata
newCode: string | undefined
newIdMap: IdMap | undefined
newMetadata: fileFormat.IdeMetadata['node'] | undefined
}
export function applyDocumentUpdates(
doc: ModuleDoc,
syncedMeta: fileFormat.Metadata,
syncedContent: string,
dataKeys: Y.YMapEvent<Uint8Array>['keys'] | null,
metadataKeys: Y.YMapEvent<NodeMetadata>['keys'] | null,
synced: EnsoFileParts,
update: ModuleUpdate,
): AppliedUpdates {
const synced = splitFileContents(syncedContent)
let codeUpdated = false
let idMapUpdated = false
if (dataKeys != null) {
for (const [key, op] of dataKeys) {
switch (op.action) {
case 'add':
case 'update': {
if (key === 'code') {
codeUpdated = true
} else if (key === 'idmap') {
idMapUpdated = true
}
break
}
const codeChanged = update.fieldsUpdated.length !== 0
let idsChanged = false
let metadataChanged = false
for (const { changes } of update.metadataUpdated) {
for (const [key] of changes) {
if (key === 'externalId') {
idsChanged = true
} else {
metadataChanged = true
}
}
if (idsChanged && metadataChanged) break
}
let newCode: string
let idMapJson: string
let metadataJson: string
let newIdMap = undefined
let newCode = undefined
let newMetadata = undefined
const allEdits: TextEdit[] = []
if (codeUpdated) {
const text = doc.getCode()
allEdits.push(...applyDiffAsTextEdits(0, synced.code, text))
newCode = text
} else {
newCode = synced.code
const syncModule = new MutableModule(doc.ydoc)
const root = syncModule.root()
assert(root != null)
if (codeChanged || idsChanged || synced.idMapJson == null) {
const { code, info } = print(root)
if (codeChanged) newCode = code
newIdMap = spanMapToIdMap(info)
}
if (idMapUpdated || synced.idMapJson == null) {
idMapJson = serializeIdMap(doc.getIdMap())
} else {
idMapJson = synced.idMapJson
}
let newMetadata = syncedMeta
if (metadataKeys != null) {
const nodeMetadata = { ...syncedMeta.ide.node }
for (const [key, op] of metadataKeys) {
switch (op.action) {
case 'delete':
delete nodeMetadata[key]
break
case 'add':
case 'update': {
const updatedMeta = doc.metadata.get(key)
const oldMeta = nodeMetadata[key] ?? {}
if (updatedMeta == null) continue
nodeMetadata[key] = {
...oldMeta,
position: {
vector: [updatedMeta.x, updatedMeta.y],
},
visualization: updatedMeta.vis
? translateVisualizationToFile(updatedMeta.vis)
: undefined,
}
break
if (codeChanged || idsChanged || metadataChanged) {
// Update the metadata object.
// Depth-first key order keeps diffs small.
newMetadata = {} satisfies fileFormat.IdeMetadata['node']
root.visitRecursiveAst((ast) => {
let pos = ast.nodeMetadata.get('position')
const vis = ast.nodeMetadata.get('visualization')
if (vis && !pos) pos = { x: 0, y: 0 }
if (pos) {
newMetadata![ast.externalId] = {
position: { vector: [Math.round(pos.x), Math.round(-pos.y)] },
visualization: vis && translateVisualizationToFile(vis),
}
}
}
// Update the metadata object without changing the original order of keys.
newMetadata = { ...syncedMeta }
newMetadata.ide = { ...syncedMeta.ide }
newMetadata.ide.node = nodeMetadata
metadataJson = json.stringify(newMetadata)
} else {
metadataJson = synced.metadataJson ?? '{}'
})
}
const newContent = combineFileParts({
code: newCode,
idMapJson,
metadataJson,
})
const oldMetaContent = syncedContent.slice(synced.code.length)
const metaContent = newContent.slice(newCode.length)
const metaStartLine = (newCode.match(/\n/g) ?? []).length
allEdits.push(...applyDiffAsTextEdits(metaStartLine, oldMetaContent, metaContent))
return {
edits: allEdits,
newContent,
newMetadata,
}
return { newCode, newIdMap, newMetadata }
}
function translateVisualizationToFile(

4
app/gui2/ydoc-server/index.ts
View File

@ -12,6 +12,7 @@ import { Server } from 'http'
import { IncomingMessage } from 'node:http'
import { parse } from 'url'
import { WebSocket, WebSocketServer } from 'ws'
import { initializeFFI } from '../shared/ast/ffi'
import { setupGatewayClient } from './ydoc'
type ConnectionData = {
@ -20,7 +21,8 @@ type ConnectionData = {
user: string
}
export function createGatewayServer(httpServer: Server) {
export async function createGatewayServer(httpServer: Server, rustFFIPath: string) {
await initializeFFI(rustFFIPath)
const wss = new WebSocketServer({ noServer: true })
wss.on('connection', (ws: WebSocket, _request: IncomingMessage, data: ConnectionData) => {
ws.on('error', onWebSocketError)

173
app/gui2/ydoc-server/languageServerSession.ts
View File

@ -1,22 +1,37 @@
import { Client, RequestManager, WebSocketTransport } from '@open-rpc/client-js'
import * as json from 'lib0/json'
import * as map from 'lib0/map'
import { ObservableV2 } from 'lib0/observable'
import * as random from 'lib0/random'
import * as Y from 'yjs'
import { splitFileContents } from '../shared/ensoFile'
import {
Ast,
MutableModule,
parseBlockWithSpans,
setExternalIds,
spanMapToIdMap,
} from '../shared/ast'
import { print } from '../shared/ast/parse'
import { EnsoFileParts, combineFileParts, splitFileContents } from '../shared/ensoFile'
import { LanguageServer, computeTextChecksum } from '../shared/languageServer'
import { Checksum, FileEdit, Path, response } from '../shared/languageServerTypes'
import { Checksum, FileEdit, Path, TextEdit, response } from '../shared/languageServerTypes'
import { exponentialBackoff, printingCallbacks } from '../shared/retry'
import {
DistributedProject,
ExternalId,
IdMap,
ModuleDoc,
type NodeMetadata,
visMetadataEquals,
type Uuid,
} from '../shared/yjsModel'
import { applyDocumentUpdates, prettyPrintDiff, translateVisualizationFromFile } from './edits'
import {
applyDiffAsTextEdits,
applyDocumentUpdates,
prettyPrintDiff,
translateVisualizationFromFile,
} from './edits'
import * as fileFormat from './fileFormat'
import { deserializeIdMap } from './serialization'
import { deserializeIdMap, serializeIdMap } from './serialization'
import { WSSharedDoc } from './ydoc'
const SOURCE_DIR = 'src'
@ -264,6 +279,9 @@ class ModulePersistence extends ObservableV2<{ removed: () => void }> {
readonly state: LsSyncState = LsSyncState.Closed
readonly lastAction = Promise.resolve()
updateToApply: Uint8Array | null = null
syncedCode: string | null = null
syncedIdMap: string | null = null
syncedMetaJson: string | null = null
syncedContent: string | null = null
syncedVersion: Checksum | null = null
syncedMeta: fileFormat.Metadata = fileFormat.tryParseMetadataOrFallback(null)
@ -398,41 +416,52 @@ class ModulePersistence extends ObservableV2<{ removed: () => void }> {
const update = this.updateToApply
this.updateToApply = null
let dataKeys: Y.YMapEvent<any>['keys'] | null = null
let metadataKeys: Y.YMapEvent<NodeMetadata>['keys'] | null = null
const observeData = (event: Y.YMapEvent<any>) => (dataKeys = event.keys)
const observeMetadata = (event: Y.YMapEvent<NodeMetadata>) => (metadataKeys = event.keys)
this.doc.data.observe(observeData)
this.doc.metadata.observe(observeMetadata)
Y.applyUpdate(this.doc.ydoc, update, 'remote')
this.doc.data.unobserve(observeData)
this.doc.metadata.unobserve(observeMetadata)
this.writeSyncedEvents(dataKeys, metadataKeys)
const syncModule = new MutableModule(this.doc.ydoc)
const moduleUpdate = syncModule.applyUpdate(update, 'remote')
if (moduleUpdate && this.syncedContent) {
const synced = splitFileContents(this.syncedContent)
const { newCode, newIdMap, newMetadata } = applyDocumentUpdates(
this.doc,
synced,
moduleUpdate,
)
this.sendLsUpdate(synced, newCode, newIdMap, newMetadata)
}
}
private writeSyncedEvents(
dataKeys: Y.YMapEvent<any>['keys'] | null,
metadataKeys: Y.YMapEvent<NodeMetadata>['keys'] | null,
private sendLsUpdate(
synced: EnsoFileParts,
newCode: string | undefined,
newIdMap: IdMap | undefined,
newMetadata: fileFormat.IdeMetadata['node'] | undefined,
) {
if (this.syncedContent == null || this.syncedVersion == null) return
if (!dataKeys && !metadataKeys) return
const { edits, newContent, newMetadata } = applyDocumentUpdates(
this.doc,
this.syncedMeta,
this.syncedContent,
dataKeys,
metadataKeys,
)
const code = newCode ?? synced.code
const newMetadataJson =
newMetadata &&
json.stringify({ ...this.syncedMeta, ide: { ...this.syncedMeta.ide, node: newMetadata } })
const newIdMapJson = newIdMap && serializeIdMap(newIdMap)
const newContent = combineFileParts({
code,
idMapJson: newIdMapJson ?? synced.idMapJson ?? '[]',
metadataJson: newMetadataJson ?? synced.metadataJson ?? '{}',
})
const edits: TextEdit[] = []
if (newCode) edits.push(...applyDiffAsTextEdits(0, synced.code, newCode))
if (newIdMap || newMetadata) {
const oldMetaContent = this.syncedContent.slice(synced.code.length)
const metaContent = newContent.slice(code.length)
const metaStartLine = (code.match(/\n/g) ?? []).length
edits.push(...applyDiffAsTextEdits(metaStartLine, oldMetaContent, metaContent))
}
const newVersion = computeTextChecksum(newContent)
if (DEBUG_LOG_SYNC) {
console.debug(' === changes === ')
console.debug('number of edits:', edits.length)
console.debug('metadata:', metadataKeys)
console.debug('data:', dataKeys)
if (edits.length > 0) {
console.debug('version:', this.syncedVersion, '->', newVersion)
console.debug('Content diff:')
@ -443,14 +472,17 @@ class ModulePersistence extends ObservableV2<{ removed: () => void }> {
this.setState(LsSyncState.WritingFile)
const execute = dataKeys != null
const execute = newCode != null || newIdMap != null
const edit: FileEdit = { path: this.path, edits, oldVersion: this.syncedVersion, newVersion }
const apply = this.ls.applyEdit(edit, execute)
const promise = apply.then(
() => {
this.syncedContent = newContent
this.syncedVersion = newVersion
this.syncedMeta = newMetadata
if (newMetadata) this.syncedMeta.ide.node = newMetadata
if (newCode) this.syncedCode = newCode
if (newIdMapJson) this.syncedIdMap = newIdMapJson
if (newMetadataJson) this.syncedMetaJson = newMetadataJson
this.setState(LsSyncState.Synchronized)
},
(error) => {
@ -460,6 +492,9 @@ class ModulePersistence extends ObservableV2<{ removed: () => void }> {
this.setState(LsSyncState.WriteError)
this.syncedContent = null
this.syncedVersion = null
this.syncedCode = null
this.syncedIdMap = null
this.syncedMetaJson = null
return this.reload()
},
)
@ -468,32 +503,72 @@ class ModulePersistence extends ObservableV2<{ removed: () => void }> {
}
private syncFileContents(content: string, version: Checksum) {
const contentsReceived = splitFileContents(content)
let unsyncedIdMap: IdMap | undefined
this.doc.ydoc.transact(() => {
const { code, idMapJson, metadataJson } = splitFileContents(content)
const { code, idMapJson, metadataJson } = contentsReceived
const metadata = fileFormat.tryParseMetadataOrFallback(metadataJson)
const nodeMeta = metadata.ide.node
const nodeMeta = Object.entries(metadata.ide.node)
const idMap = idMapJson ? deserializeIdMap(idMapJson) : new IdMap()
this.doc.setIdMap(idMap)
const keysToDelete = new Set(this.doc.metadata.keys())
for (const [id, meta] of Object.entries(nodeMeta)) {
if (typeof id !== 'string') continue
const formattedMeta: NodeMetadata = {
x: meta.position.vector[0],
y: meta.position.vector[1],
vis: (meta.visualization && translateVisualizationFromFile(meta.visualization)) ?? null,
}
keysToDelete.delete(id)
this.doc.metadata.set(id, formattedMeta)
let parsedSpans
const syncModule = new MutableModule(this.doc.ydoc)
if (code !== this.syncedCode) {
const { root, spans } = parseBlockWithSpans(code, syncModule)
syncModule.syncRoot(root)
parsedSpans = spans
}
for (const id of keysToDelete) this.doc.metadata.delete(id)
const astRoot = syncModule.root()
if (!astRoot) return
if ((code !== this.syncedCode || idMapJson !== this.syncedIdMap) && idMapJson) {
const idMap = deserializeIdMap(idMapJson)
const spans = parsedSpans ?? print(astRoot).info
const newExternalIds = setExternalIds(syncModule, spans, idMap)
if (newExternalIds !== 0) {
if (code !== this.syncedCode) {
unsyncedIdMap = spanMapToIdMap(spans)
} else {
console.warn(
`The LS sent an IdMap-only edit that is missing ${newExternalIds} of our expected ASTs.`,
)
}
}
}
if (
(code !== this.syncedCode ||
idMapJson !== this.syncedIdMap ||
metadataJson !== this.syncedMetaJson) &&
nodeMeta.length !== 0
) {
const externalIdToAst = new Map<ExternalId, Ast>()
astRoot.visitRecursiveAst((ast) => {
if (!externalIdToAst.has(ast.externalId)) externalIdToAst.set(ast.externalId, ast)
})
const missing = new Set<string>()
for (const [id, meta] of nodeMeta) {
if (typeof id !== 'string') continue
const ast = externalIdToAst.get(id as ExternalId)
if (!ast) {
missing.add(id)
continue
}
const metadata = syncModule.getVersion(ast).mutableNodeMetadata()
const oldPos = metadata.get('position')
const newPos = { x: meta.position.vector[0], y: -meta.position.vector[1] }
if (oldPos?.x !== newPos.x || oldPos?.y !== newPos.y) metadata.set('position', newPos)
const oldVis = metadata.get('visualization')
const newVis = meta.visualization && translateVisualizationFromFile(meta.visualization)
if (!visMetadataEquals(newVis, oldVis)) metadata.set('visualization', newVis)
}
}
this.syncedCode = code
this.syncedIdMap = unsyncedIdMap ? null : idMapJson
this.syncedContent = content
this.syncedVersion = version
this.syncedMeta = metadata
this.doc.setCode(code)
this.syncedMetaJson = metadataJson
}, 'file')
if (unsyncedIdMap) this.sendLsUpdate(contentsReceived, undefined, unsyncedIdMap, undefined)
}
async close() {

32
app/ide-desktop/lib/client/src/bin/server.ts
View File

@ -96,17 +96,27 @@ export class Server {
handler: this.process.bind(this),
},
(err, { http: httpServer }) => {
if (err) {
logger.error(`Error creating server:`, err.http)
reject(err)
}
// Prepare the YDoc server access point for the new Vue-based GUI.
if (httpServer) {
ydocServer.createGatewayServer(httpServer)
}
logger.log(`Server started on port ${this.config.port}.`)
logger.log(`Serving files from '${path.join(process.cwd(), this.config.dir)}'.`)
resolve()
void (async () => {
if (err) {
logger.error(`Error creating server:`, err.http)
reject(err)
}
// Prepare the YDoc server access point for the new Vue-based GUI.
if (httpServer) {
await ydocServer.createGatewayServer(
httpServer,
// TODO[ao]: This is very ugly quickfix to make our rust-ffi WASM
// working both in browser and in ydocs server. Doing it properly
// is tracked in https://github.com/enso-org/enso/issues/8931
path.join(paths.ASSETS_PATH, 'assets', 'rust_ffi_bg-c353f976.wasm')
)
}
logger.log(`Server started on port ${this.config.port}.`)
logger.log(
`Serving files from '${path.join(process.cwd(), this.config.dir)}'.`
)
resolve()
})()
}
)
})

305
package-lock.json generated
View File

@ -54,6 +54,7 @@
"pinia": "^2.1.6",
"postcss-inline-svg": "^6.0.0",
"postcss-nesting": "^12.0.1",
"rimraf": "^5.0.5",
"semver": "^7.5.4",
"sucrase": "^3.34.0",
"vue": "^3.3.4",
@ -123,6 +124,52 @@
"vue-tsc": "^1.8.8"
}
},
"app/gui2/node_modules/glob": {
"version": "10.3.10",
"resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz",
"integrity": "sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==",
"dependencies": {
"foreground-child": "^3.1.0",
"jackspeak": "^2.3.5",
"minimatch": "^9.0.1",
"minipass": "^5.0.0 || ^6.0.2 || ^7.0.0",
"path-scurry": "^1.10.1"
},
"bin": {
"glob": "dist/esm/bin.mjs"
},
"engines": {
"node": ">=16 || 14 >=14.17"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"app/gui2/node_modules/minipass": {
"version": "7.0.4",
"resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
"integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==",
"engines": {
"node": ">=16 || 14 >=14.17"
}
},
"app/gui2/node_modules/rimraf": {
"version": "5.0.5",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz",
"integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==",
"dependencies": {
"glob": "^10.3.7"
},
"bin": {
"rimraf": "dist/esm/bin.mjs"
},
"engines": {
"node": ">=14"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"app/ide-desktop": {
"name": "enso-ide-desktop",
"version": "1.0.0",
@ -3237,6 +3284,95 @@
}
}
},
"node_modules/@isaacs/cliui": {
"version": "8.0.2",
"resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz",
"integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==",
"dependencies": {
"string-width": "^5.1.2",
"string-width-cjs": "npm:string-width@^4.2.0",
"strip-ansi": "^7.0.1",
"strip-ansi-cjs": "npm:strip-ansi@^6.0.1",
"wrap-ansi": "^8.1.0",
"wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0"
},
"engines": {
"node": ">=12"
}
},
"node_modules/@isaacs/cliui/node_modules/ansi-regex": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz",
"integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/chalk/ansi-regex?sponsor=1"
}
},
"node_modules/@isaacs/cliui/node_modules/ansi-styles": {
"version": "6.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz",
"integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/@isaacs/cliui/node_modules/emoji-regex": {
"version": "9.2.2",
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz",
"integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="
},
"node_modules/@isaacs/cliui/node_modules/string-width": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz",
"integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==",
"dependencies": {
"eastasianwidth": "^0.2.0",
"emoji-regex": "^9.2.2",
"strip-ansi": "^7.0.1"
},
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/@isaacs/cliui/node_modules/strip-ansi": {
"version": "7.1.0",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz",
"integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==",
"dependencies": {
"ansi-regex": "^6.0.1"
},
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/chalk/strip-ansi?sponsor=1"
}
},
"node_modules/@isaacs/cliui/node_modules/wrap-ansi": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz",
"integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==",
"dependencies": {
"ansi-styles": "^6.1.0",
"string-width": "^5.0.1",
"strip-ansi": "^7.0.1"
},
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/chalk/wrap-ansi?sponsor=1"
}
},
"node_modules/@istanbuljs/schema": {
"version": "0.1.3",
"dev": true,
@ -3526,6 +3662,15 @@
"pinia": ">=2.1.5"
}
},
"node_modules/@pkgjs/parseargs": {
"version": "0.11.0",
"resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz",
"integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==",
"optional": true,
"engines": {
"node": ">=14"
}
},
"node_modules/@pkgr/utils": {
"version": "2.4.2",
"dev": true,
@ -6089,7 +6234,6 @@
},
"node_modules/brace-expansion": {
"version": "2.0.1",
"dev": true,
"license": "MIT",
"dependencies": {
"balanced-match": "^1.0.0"
@ -6959,7 +7103,6 @@
},
"node_modules/cross-spawn": {
"version": "7.0.3",
"dev": true,
"license": "MIT",
"dependencies": {
"path-key": "^3.1.0",
@ -7948,6 +8091,11 @@
"dev": true,
"license": "BSD-3-Clause"
},
"node_modules/eastasianwidth": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz",
"integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA=="
},
"node_modules/easy-stack": {
"version": "1.0.1",
"dev": true,
@ -9528,6 +9676,32 @@
"is-callable": "^1.1.3"
}
},
"node_modules/foreground-child": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz",
"integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==",
"dependencies": {
"cross-spawn": "^7.0.0",
"signal-exit": "^4.0.1"
},
"engines": {
"node": ">=14"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/foreground-child/node_modules/signal-exit": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz",
"integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==",
"engines": {
"node": ">=14"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/forever-agent": {
"version": "0.6.1",
"dev": true,
@ -11111,7 +11285,6 @@
},
"node_modules/isexe": {
"version": "2.0.0",
"dev": true,
"license": "ISC"
},
"node_modules/isomorphic-fetch": {
@ -11241,6 +11414,23 @@
"set-function-name": "^2.0.1"
}
},
"node_modules/jackspeak": {
"version": "2.3.6",
"resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz",
"integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==",
"dependencies": {
"@isaacs/cliui": "^8.0.2"
},
"engines": {
"node": ">=14"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
},
"optionalDependencies": {
"@pkgjs/parseargs": "^0.11.0"
}
},
"node_modules/jake": {
"version": "10.8.7",
"dev": true,
@ -12255,7 +12445,6 @@
},
"node_modules/minimatch": {
"version": "9.0.1",
"dev": true,
"license": "ISC",
"dependencies": {
"brace-expansion": "^2.0.1"
@ -13296,7 +13485,6 @@
},
"node_modules/path-key": {
"version": "3.1.1",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=8"
@ -13307,6 +13495,37 @@
"dev": true,
"license": "MIT"
},
"node_modules/path-scurry": {
"version": "1.10.1",
"resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.1.tgz",
"integrity": "sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ==",
"dependencies": {
"lru-cache": "^9.1.1 || ^10.0.0",
"minipass": "^5.0.0 || ^6.0.2 || ^7.0.0"
},
"engines": {
"node": ">=16 || 14 >=14.17"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/path-scurry/node_modules/lru-cache": {
"version": "10.2.0",
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.0.tgz",
"integrity": "sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==",
"engines": {
"node": "14 || >=16.14"
}
},
"node_modules/path-scurry/node_modules/minipass": {
"version": "7.0.4",
"resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
"integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==",
"engines": {
"node": ">=16 || 14 >=14.17"
}
},
"node_modules/path-type": {
"version": "4.0.0",
"license": "MIT",
@ -14941,7 +15160,6 @@
},
"node_modules/shebang-command": {
"version": "2.0.0",
"dev": true,
"license": "MIT",
"dependencies": {
"shebang-regex": "^3.0.0"
@ -14952,7 +15170,6 @@
},
"node_modules/shebang-regex": {
"version": "3.0.0",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=8"
@ -15361,6 +15578,20 @@
"node": ">=8"
}
},
"node_modules/string-width-cjs": {
"name": "string-width",
"version": "4.2.3",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz",
"integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==",
"dependencies": {
"emoji-regex": "^8.0.0",
"is-fullwidth-code-point": "^3.0.0",
"strip-ansi": "^6.0.1"
},
"engines": {
"node": ">=8"
}
},
"node_modules/string.prototype.matchall": {
"version": "4.0.10",
"dev": true,
@ -15448,6 +15679,18 @@
"node": ">=8"
}
},
"node_modules/strip-ansi-cjs": {
"name": "strip-ansi",
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz",
"integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==",
"dependencies": {
"ansi-regex": "^5.0.1"
},
"engines": {
"node": ">=8"
}
},
"node_modules/strip-bom": {
"version": "3.0.0",
"dev": true,
@ -17981,7 +18224,6 @@
},
"node_modules/which": {
"version": "2.0.2",
"dev": true,
"license": "ISC",
"dependencies": {
"isexe": "^2.0.0"
@ -18106,6 +18348,53 @@
"url": "https://github.com/chalk/wrap-ansi?sponsor=1"
}
},
"node_modules/wrap-ansi-cjs": {
"name": "wrap-ansi",
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
"dependencies": {
"ansi-styles": "^4.0.0",
"string-width": "^4.1.0",
"strip-ansi": "^6.0.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/chalk/wrap-ansi?sponsor=1"
}
},
"node_modules/wrap-ansi-cjs/node_modules/ansi-styles": {
"version": "4.3.0",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
"dependencies": {
"color-convert": "^2.0.1"
},
"engines": {
"node": ">=8"
},
"funding": {
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
}
},
"node_modules/wrap-ansi-cjs/node_modules/color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
"dependencies": {
"color-name": "~1.1.4"
},
"engines": {
"node": ">=7.0.0"
}
},
"node_modules/wrap-ansi-cjs/node_modules/color-name": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
},
"node_modules/wrap-ansi/node_modules/ansi-styles": {
"version": "4.3.0",
"license": "MIT",