New markdown editor (#11469)

Implements #11240.

https://github.com/user-attachments/assets/4d2f8021-3e0f-4d39-95df-bcd72bf7545b

# Important Notes
- Fix a Yjs document corruption bug caused by `DeepReadonly` being replaced by a stub; introduce a `DeepReadonly` implementation without Vue dependency.
- Fix right panel sizing when code editor is open.
- Fix right panel slide-in animation.
- `Ast.Function` renamed to `Ast.FunctionDef`.

CHANGELOG.md

@@ -22,6 +22,8 @@
 - [Table Input Widget has now a limit of 256 cells.][11448]
 - [Added an error message screen displayed when viewing a deleted
   component.][11452]
+- [New documentation editor provides improved Markdown editing experience, and
+  paves the way for new documentation features.][11469]
 
 [11151]: https://github.com/enso-org/enso/pull/11151
 [11271]: https://github.com/enso-org/enso/pull/11271
@@ -37,6 +39,7 @@
 [11447]: https://github.com/enso-org/enso/pull/11447
 [11448]: https://github.com/enso-org/enso/pull/11448
 [11452]: https://github.com/enso-org/enso/pull/11452
+[11469]: https://github.com/enso-org/enso/pull/11469
 
 #### Enso Standard Library
 

app/common/package.json

@@ -18,6 +18,8 @@
     "./src/utilities/data/dateTime": "./src/utilities/data/dateTime.ts",
     "./src/utilities/data/newtype": "./src/utilities/data/newtype.ts",
     "./src/utilities/data/object": "./src/utilities/data/object.ts",
+    "./src/utilities/data/string": "./src/utilities/data/string.ts",
+    "./src/utilities/data/iter": "./src/utilities/data/iter.ts",
     "./src/utilities/style/tabBar": "./src/utilities/style/tabBar.ts",
     "./src/utilities/uniqueString": "./src/utilities/uniqueString.ts",
     "./src/text": "./src/text/index.ts",
@@ -37,6 +39,7 @@
     "@tanstack/query-persist-client-core": "^5.54.0",
     "@tanstack/vue-query": ">= 5.54.0 < 5.56.0",
     "idb-keyval": "^6.2.1",
+    "lib0": "^0.2.85",
     "react": "^18.3.1",
     "vitest": "^1.3.1",
     "vue": "^3.5.2"

app/common/src/utilities/data/__tests__/iterator.test.ts

@@ -0,0 +1,146 @@
+import { expect, test } from 'vitest'
+import * as iter from '../iter'
+
+interface IteratorCase<T> {
+  iterable: Iterable<T>
+  soleValue: T | undefined
+  first: T | undefined
+  last: T | undefined
+  count: number
+}
+
+function makeCases(): IteratorCase<unknown>[] {
+  return [
+    {
+      iterable: iter.empty(),
+      soleValue: undefined,
+      first: undefined,
+      last: undefined,
+      count: 0,
+    },
+    {
+      iterable: iter.chain(iter.empty(), iter.empty()),
+      soleValue: undefined,
+      first: undefined,
+      last: undefined,
+      count: 0,
+    },
+    {
+      iterable: iter.chain(iter.empty(), ['a'], iter.empty()),
+      soleValue: 'a',
+      first: 'a',
+      last: 'a',
+      count: 1,
+    },
+    {
+      iterable: iter.range(10, 11),
+      soleValue: 10,
+      first: 10,
+      last: 10,
+      count: 1,
+    },
+    {
+      iterable: iter.range(10, 20),
+      soleValue: undefined,
+      first: 10,
+      last: 19,
+      count: 10,
+    },
+    {
+      iterable: iter.range(20, 10),
+      soleValue: undefined,
+      first: 20,
+      last: 11,
+      count: 10,
+    },
+    {
+      iterable: [],
+      soleValue: undefined,
+      first: undefined,
+      last: undefined,
+      count: 0,
+    },
+    {
+      iterable: ['a'],
+      soleValue: 'a',
+      first: 'a',
+      last: 'a',
+      count: 1,
+    },
+    {
+      iterable: ['a', 'b'],
+      soleValue: undefined,
+      first: 'a',
+      last: 'b',
+      count: 2,
+    },
+    {
+      iterable: iter.filterDefined([undefined, 'a', undefined, 'b', undefined]),
+      soleValue: undefined,
+      first: 'a',
+      last: 'b',
+      count: 2,
+    },
+    {
+      iterable: iter.filter([7, 'a', 8, 'b', 9], el => typeof el === 'string'),
+      soleValue: undefined,
+      first: 'a',
+      last: 'b',
+      count: 2,
+    },
+    {
+      iterable: iter.zip(['a', 'b'], iter.range(1, 2)),
+      soleValue: ['a', 1],
+      first: ['a', 1],
+      last: ['a', 1],
+      count: 1,
+    },
+    {
+      iterable: iter.zip(['a', 'b'], iter.range(1, 3)),
+      soleValue: undefined,
+      first: ['a', 1],
+      last: ['b', 2],
+      count: 2,
+    },
+    {
+      iterable: iter.zip(['a', 'b'], iter.range(1, 4)),
+      soleValue: undefined,
+      first: ['a', 1],
+      last: ['b', 2],
+      count: 2,
+    },
+    {
+      iterable: iter.zipLongest(['a', 'b'], iter.range(1, 2)),
+      soleValue: undefined,
+      first: ['a', 1],
+      last: ['b', undefined],
+      count: 2,
+    },
+    {
+      iterable: iter.zipLongest(['a', 'b'], iter.range(1, 3)),
+      soleValue: undefined,
+      first: ['a', 1],
+      last: ['b', 2],
+      count: 2,
+    },
+    {
+      iterable: iter.zipLongest(['a', 'b'], iter.range(1, 4)),
+      soleValue: undefined,
+      first: ['a', 1],
+      last: [undefined, 3],
+      count: 3,
+    },
+  ]
+}
+
+test.each(makeCases())('tryGetSoleValue: case %#', ({ iterable, soleValue }) => {
+  expect(iter.tryGetSoleValue(iterable)).toEqual(soleValue)
+})
+
+test.each(makeCases())('last: case %#', ({ iterable, last }) => {
+  expect(iter.last(iterable)).toEqual(last)
+})
+
+test.each(makeCases())('count: case %#', ({ iterable, count }) => {
+  expect(iter.count(iterable)).toEqual(count)
+})

app/common/src/utilities/data/iter.ts

@@ -1,4 +1,30 @@
-/** @file Functions for manipulating {@link Iterable}s. */
+/** @file Utilities for manipulating {@link Iterator}s and {@link Iterable}s. */
+
+import { iteratorFilter, mapIterator } from 'lib0/iterator'
+
+/** Similar to {@link Array.prototype.reduce|}, but consumes elements from any iterable. */
+export function reduce<T, A>(
+  iterable: Iterable<T>,
+  f: (accumulator: A, element: T) => A,
+  initialAccumulator: A,
+): A {
+  const iterator = iterable[Symbol.iterator]()
+  let accumulator = initialAccumulator
+  let result = iterator.next()
+  while (!result.done) {
+    accumulator = f(accumulator, result.value)
+    result = iterator.next()
+  }
+  return accumulator
+}
+
+/**
+ * Iterates the provided iterable, returning the number of elements it yielded. Note that if the input is an iterator,
+ * it will be consumed.
+ */
+export function count(it: Iterable<unknown>): number {
+  return reduce(it, a => a + 1, 0)
+}
 
 /** An iterable with zero elements. */
 export function* empty(): Generator<never> {}
@@ -26,22 +52,17 @@ export function* range(start: number, stop: number, step = start <= stop ? 1 : -
   }
 }
 
-/**
- * Return an {@link Iterable} that `yield`s values that are the result of calling the given
- * function on the next value of the given source iterable.
- */
-export function* map<T, U>(iter: Iterable<T>, map: (value: T) => U): IterableIterator<U> {
-  for (const value of iter) {
-    yield map(value)
-  }
+/** @returns An iterator that yields the results of applying the given function to each value of the given iterable. */
+export function map<T, U>(it: Iterable<T>, f: (value: T) => U): IterableIterator<U> {
+  return mapIterator(it[Symbol.iterator](), f)
 }
 
 /**
  * Return an {@link Iterable} that `yield`s only the values from the given source iterable
  * that pass the given predicate.
  */
-export function* filter<T>(iter: Iterable<T>, include: (value: T) => boolean): IterableIterator<T> {
-  for (const value of iter) if (include(value)) yield value
+export function filter<T>(iter: Iterable<T>, include: (value: T) => boolean): IterableIterator<T> {
+  return iteratorFilter(iter[Symbol.iterator](), include)
 }
 
 /**
@@ -141,3 +162,45 @@ export class Resumable<T> {
     }
   }
 }
+
+/** Returns an iterator that yields the values of the provided iterator that are not strictly-equal to `undefined`. */
+export function* filterDefined<T>(iterable: Iterable<T | undefined>): IterableIterator<T> {
+  for (const value of iterable) {
+    if (value !== undefined) yield value
+  }
+}
+
+/**
+ * Returns whether the predicate returned `true` for all values yielded by the provided iterator. Short-circuiting.
+ * Returns `true` if the iterator doesn't yield any values.
+ */
+export function every<T>(iter: Iterable<T>, f: (value: T) => boolean): boolean {
+  for (const value of iter) if (!f(value)) return false
+  return true
+}
+
+/** Return the first element returned by the iterable which meets the condition. */
+export function find<T>(iter: Iterable<T>, f: (value: T) => boolean): T | undefined {
+  for (const value of iter) {
+    if (f(value)) return value
+  }
+  return undefined
+}
+
+/** Returns the first element yielded by the iterable. */
+export function first<T>(iterable: Iterable<T>): T | undefined {
+  const iterator = iterable[Symbol.iterator]()
+  const result = iterator.next()
+  return result.done ? undefined : result.value
+}
+
+/**
+ * Return last element returned by the iterable.
+ * NOTE: Linear complexity. This function always visits the whole iterable. Using this with an
+ * infinite generator will cause an infinite loop.
+ */
+export function last<T>(iter: Iterable<T>): T | undefined {
+  let last
+  for (const el of iter) last = el
+  return last
+}

app/common/src/utilities/data/object.ts

@@ -162,3 +162,24 @@ export type ExtractKeys<T, U> = {
 
 /** An instance method of the given type. */
 export type MethodOf<T> = (this: T, ...args: never) => unknown
+
+// ===================
+// === useObjectId ===
+// ===================
+
+/** Composable providing support for managing object identities. */
+export function useObjectId() {
+  let lastId = 0
+  const idNumbers = new WeakMap<object, number>()
+  /** @returns A value that can be used to compare object identity. */
+  function objectId(o: object): number {
+    const id = idNumbers.get(o)
+    if (id == null) {
+      lastId += 1
+      idNumbers.set(o, lastId)
+      return lastId
+    }
+    return id
+  }
+  return { objectId }
+}

app/common/src/utilities/data/string.ts

@@ -0,0 +1,2 @@
+/** See http://www.unicode.org/reports/tr18/#Line_Boundaries */
+export const LINE_BOUNDARIES = /\r\n|[\n\v\f\r\x85\u2028\u2029]/g

app/gui/e2e/project-view/locate.ts

@@ -84,7 +84,7 @@ export const addNewNodeButton = componentLocator('.PlusButton')
 export const componentBrowser = componentLocator('.ComponentBrowser')
 export const nodeOutputPort = componentLocator('.outputPortHoverArea')
 export const smallPlusButton = componentLocator('.SmallPlusButton')
-export const lexicalContent = componentLocator('.LexicalContent')
+export const editorRoot = componentLocator('.EditorRoot')
 
 /**
  * A not-selected variant of Component Browser Entry.

app/gui/e2e/project-view/rightPanel.spec.ts

@@ -1,6 +1,6 @@
 import { expect, test } from 'playwright/test'
 import * as actions from './actions'
-import { mockMethodCallInfo } from './expressionUpdates'
+import { mockCollapsedFunctionInfo, mockMethodCallInfo } from './expressionUpdates'
 import { CONTROL_KEY } from './keyboard'
 import * as locate from './locate'
 
@@ -13,7 +13,7 @@ test('Main method documentation', async ({ page }) => {
   await expect(locate.rightDock(page)).toBeVisible()
 
   // Right-dock displays main method documentation.
-  await expect(locate.lexicalContent(locate.rightDock(page))).toHaveText('The main method')
+  await expect(locate.editorRoot(locate.rightDock(page))).toHaveText('The main method')
 
   // Documentation hotkey closes right-dock.p
   await page.keyboard.press(`${CONTROL_KEY}+D`)
@@ -70,3 +70,20 @@ test('Component help', async ({ page }) => {
   await locate.graphNodeByBinding(page, 'data').click()
   await expect(locate.rightDock(page)).toHaveText(/Reads a file into Enso/)
 })
+
+test('Documentation reflects entered function', async ({ page }) => {
+  await actions.goToGraph(page)
+
+  // Open the panel
+  await expect(locate.rightDock(page)).toBeHidden()
+  await page.keyboard.press(`${CONTROL_KEY}+D`)
+  await expect(locate.rightDock(page)).toBeVisible()
+
+  // Enter the collapsed function
+  await mockCollapsedFunctionInfo(page, 'final', 'func1')
+  await locate.graphNodeByBinding(page, 'final').dblclick()
+  await expect(locate.navBreadcrumb(page)).toHaveText(['Mock Project', 'func1'])
+
+  // Editor should contain collapsed function's docs
+  await expect(locate.editorRoot(locate.rightDock(page))).toHaveText('A collapsed function')
+})

app/gui/package.json

@@ -83,22 +83,18 @@
     "babel-plugin-react-compiler": "19.0.0-beta-9ee70a1-20241017",
     "@codemirror/commands": "^6.6.0",
     "@codemirror/language": "^6.10.2",
+    "@codemirror/lang-markdown": "^v6.3.0",
     "@codemirror/lint": "^6.8.1",
     "@codemirror/search": "^6.5.6",
     "@codemirror/state": "^6.4.1",
     "@codemirror/view": "^6.28.3",
     "@fast-check/vitest": "^0.0.8",
     "@floating-ui/vue": "^1.0.6",
-    "@lexical/code": "^0.16.0",
     "@lexical/link": "^0.16.0",
-    "@lexical/list": "^0.16.0",
-    "@lexical/markdown": "^0.16.0",
     "@lexical/plain-text": "^0.16.0",
-    "@lexical/rich-text": "^0.16.0",
-    "@lexical/selection": "^0.16.0",
-    "@lexical/table": "^0.16.0",
     "@lexical/utils": "^0.16.0",
     "@lezer/common": "^1.1.0",
+    "@lezer/markdown": "^1.3.1",
     "@lezer/highlight": "^1.1.6",
     "@noble/hashes": "^1.4.0",
     "@vueuse/core": "^10.4.1",

app/gui/src/project-view/assets/base.css

@@ -85,7 +85,7 @@
   /* Resize handle override for the visualization container. */
   --visualization-resize-handle-inside: 3px;
   --visualization-resize-handle-outside: 3px;
-  --right-dock-default-width: 40%;
+  --right-dock-default-width: 40vw;
   --code-editor-default-height: 30%;
   --scrollbar-scrollable-opacity: 100%;
 }

app/gui/src/project-view/components/CodeEditor.vue

@@ -1,14 +1,16 @@
 <script setup lang="ts">
 import type { ChangeSet, Diagnostic, Highlighter } from '@/components/CodeEditor/codemirror'
+import EditorRoot from '@/components/EditorRoot.vue'
 import { useGraphStore, type NodeId } from '@/stores/graph'
 import { useProjectStore } from '@/stores/project'
 import { useSuggestionDbStore } from '@/stores/suggestionDatabase'
 import { useAutoBlur } from '@/util/autoBlur'
-import { chain } from '@/util/data/iterable'
 import { unwrap } from '@/util/data/result'
 import { qnJoin, tryQualifiedName } from '@/util/qualifiedName'
 import { EditorSelection } from '@codemirror/state'
+import * as iter from 'enso-common/src/utilities/data/iter'
 import { createDebouncer } from 'lib0/eventloop'
+import type { ComponentInstance } from 'vue'
 import { computed, onMounted, onUnmounted, ref, shallowRef, watch, watchEffect } from 'vue'
 import { MutableModule } from 'ydoc-shared/ast'
 import { textChangeToEdits, type SourceRangeEdit } from 'ydoc-shared/util/data/text'
@@ -40,7 +42,8 @@ const {
 const projectStore = useProjectStore()
 const graphStore = useGraphStore()
 const suggestionDbStore = useSuggestionDbStore()
-const rootElement = ref<HTMLElement>()
+const editorRoot = ref<ComponentInstance<typeof EditorRoot>>()
+const rootElement = computed(() => editorRoot.value?.rootElement)
 useAutoBlur(rootElement)
 
 const executionContextDiagnostics = shallowRef<Diagnostic[]>([])
@@ -63,7 +66,7 @@ const expressionUpdatesDiagnostics = computed(() => {
   const panics = updates.type.reverseLookup('Panic')
   const errors = updates.type.reverseLookup('DataflowError')
   const diagnostics: Diagnostic[] = []
-  for (const externalId of chain(panics, errors)) {
+  for (const externalId of iter.chain(panics, errors)) {
     const update = updates.get(externalId)
     if (!update) continue
     const astId = graphStore.db.idFromExternal(externalId)
@@ -322,25 +325,11 @@ onMounted(() => {
 </script>
 
 <template>
-  <div
-    ref="rootElement"
-    class="CodeEditor"
-    @keydown.arrow-left.stop
-    @keydown.arrow-right.stop
-    @keydown.arrow-up.stop
-    @keydown.arrow-down.stop
-    @keydown.enter.stop
-    @keydown.backspace.stop
-    @keydown.delete.stop
-    @wheel.stop.passive
-    @contextmenu.stop
-  ></div>
+  <EditorRoot ref="editorRoot" class="CodeEditor" />
 </template>
 
 <style scoped>
 .CodeEditor {
-  width: 100%;
-  height: 100%;
   font-family: var(--font-mono);
   backdrop-filter: var(--blur-app-bg);
   background-color: rgba(255, 255, 255, 0.9);
@@ -348,13 +337,13 @@ onMounted(() => {
   border: 1px solid rgba(255, 255, 255, 0.4);
 }
 
-:deep(.ͼ1 .cm-scroller) {
+:deep(.cm-scroller) {
   font-family: var(--font-mono);
   /* Prevent touchpad back gesture, which can be triggered while panning. */
   overscroll-behavior: none;
 }
 
-.CodeEditor :deep(.cm-editor) {
+:deep(.cm-editor) {
   position: relative;
   width: 100%;
   height: 100%;
@@ -366,11 +355,11 @@ onMounted(() => {
   transition: outline 0.1s ease-in-out;
 }
 
-.CodeEditor :deep(.cm-focused) {
+:deep(.cm-focused) {
   outline: 1px solid rgba(0, 0, 0, 0.5);
 }
 
-.CodeEditor :deep(.cm-tooltip-hover) {
+:deep(.cm-tooltip-hover) {
   padding: 4px;
   border-radius: 4px;
   border: 1px solid rgba(0, 0, 0, 0.4);
@@ -384,7 +373,7 @@ onMounted(() => {
   }
 }
 
-.CodeEditor :deep(.cm-gutters) {
+:deep(.cm-gutters) {
   border-radius: 3px 0 0 3px;
   min-width: 32px;
 }

app/gui/src/project-view/components/CodeEditor/codemirror.ts

@@ -43,8 +43,8 @@ import {
 } from '@lezer/common'
 import { styleTags, tags } from '@lezer/highlight'
 import { EditorView } from 'codemirror'
+import * as iter from 'enso-common/src/utilities/data/iter'
 import type { Diagnostic as LSDiagnostic } from 'ydoc-shared/languageServerTypes'
-import { tryGetSoleValue } from 'ydoc-shared/util/data/iterable'
 import type { SourceRangeEdit } from 'ydoc-shared/util/data/text'
 
 /** TODO: Add docs */
@@ -124,7 +124,7 @@ function astToCodeMirrorTree(
   const [start, end] = ast.span()
   const children = ast.children()
 
-  const childrenToConvert = tryGetSoleValue(children)?.isToken() ? [] : children
+  const childrenToConvert = iter.tryGetSoleValue(children)?.isToken() ? [] : children
 
   const tree = new Tree(
     nodeSet.types[ast.inner.type + (ast.isToken() ? RawAst.Tree.typeNames.length : 0)]!,

app/gui/src/project-view/components/ColorPickerMenu.vue

@@ -3,9 +3,8 @@ import ColorRing from '@/components/ColorRing.vue'
 import { injectNodeColors } from '@/providers/graphNodeColors'
 import { injectGraphSelection } from '@/providers/graphSelection'
 import { useGraphStore, type NodeId } from '@/stores/graph'
-import { filterDefined } from '@/util/data/iterable'
+import * as iter from 'enso-common/src/utilities/data/iter'
 import { ref } from 'vue'
-import { tryGetSoleValue } from 'ydoc-shared/util/data/iterable'
 
 const emit = defineEmits<{
   close: []
@@ -16,9 +15,9 @@ const selection = injectGraphSelection()
 const graphStore = useGraphStore()
 
 const displayedColors = new Set<string>(
-  filterDefined(Array.from(selection.selected, (node) => getNodeColor(node))),
+  iter.filterDefined(iter.map(selection.selected, getNodeColor)),
 )
-const currentColor = ref<string | undefined>(tryGetSoleValue(displayedColors.values()))
+const currentColor = ref<string | undefined>(iter.tryGetSoleValue(displayedColors.values()))
 
 const editedNodeInitialColors = new Map<NodeId, string | undefined>()
 

app/gui/src/project-view/components/ColorRing/gradient.ts

@@ -1,5 +1,5 @@
 import { ensoColor, formatCssColor, normalizeHue } from '@/util/colors'
-import { Resumable } from 'ydoc-shared/util/data/iterable'
+import * as iter from 'enso-common/src/utilities/data/iter'
 
 export interface FixedRange {
   start: number
@@ -98,7 +98,7 @@ export function gradientPoints(
 ): GradientPoint[] {
   const points = new Array<GradientPoint>()
   const interpolationPoint = (angle: number) => ({ hue: angle, angle })
-  const fixedRangeIter = new Resumable(inputRanges)
+  const fixedRangeIter = new iter.Resumable(inputRanges)
   const min = Math.max(3, Math.round(minStops ?? 0))
   for (let i = 0; i < min; i++) {
     const angle = i / (min - 1)

app/gui/src/project-view/components/ComponentBrowser/ComponentEditor.vue

@@ -4,7 +4,7 @@ import AutoSizedInput, { type Range } from '@/components/widgets/AutoSizedInput.
 import type { useNavigator } from '@/composables/navigator'
 import type { Icon } from '@/util/iconName'
 import { computed, ref, watch, type DeepReadonly } from 'vue'
-import { ComponentExposed } from 'vue-component-type-helpers'
+import type { ComponentExposed } from 'vue-component-type-helpers'
 
 const content = defineModel<DeepReadonly<{ text: string; selection: Range | undefined }>>({
   required: true,

app/gui/src/project-view/components/ComponentBrowser/__tests__/placement.test.ts

@@ -7,11 +7,10 @@ import {
   type Environment,
   type InputNodeEnvironment,
 } from '@/components/ComponentBrowser/placement'
-import * as iterable from '@/util/data/iterable'
-import { chain, map, range } from '@/util/data/iterable'
 import { Rect } from '@/util/data/rect'
 import { Vec2 } from '@/util/data/vec2'
 import { fc, test as fcTest } from '@fast-check/vitest'
+import * as iter from 'enso-common/src/utilities/data/iter'
 import { describe, expect, test } from 'vitest'
 
 // Vue playground to visually inspect failing fuzz cases:
@@ -42,7 +41,7 @@ describe('Non dictated placement', () => {
     return {
       screenBounds,
       nodeRects,
-      selectedNodeRects: iterable.empty(),
+      selectedNodeRects: iter.empty(),
     }
   }
 
@@ -84,53 +83,53 @@ describe('Non dictated placement', () => {
     // === Multiple node tests ===
     {
       desc: 'Multiple nodes',
-      nodes: map(range(0, 1001, 20), rectAtX(1050)),
+      nodes: iter.map(iter.range(0, 1001, 20), rectAtX(1050)),
       pos: new Vec2(1090, 1044),
     },
     {
       desc: 'Multiple nodes with gap',
-      nodes: map(range(1000, -1, -20), rectAtX(1050)),
+      nodes: iter.map(iter.range(1000, -1, -20), rectAtX(1050)),
       pos: new Vec2(1090, 1044),
     },
     {
       desc: 'Multiple nodes with gap 2',
-      nodes: chain(
-        map(range(500, 901, 20), rectAtX(1050)),
-        map(range(1000, 1501, 20), rectAtX(1050)),
+      nodes: iter.chain(
+        iter.map(iter.range(500, 901, 20), rectAtX(1050)),
+        iter.map(iter.range(1000, 1501, 20), rectAtX(1050)),
       ),
       pos: new Vec2(1090, 944),
     },
     {
       desc: 'Multiple nodes with gap (just big enough)',
-      nodes: map(range(690, 1500, 88), rectAtX(1050)),
+      nodes: iter.map(iter.range(690, 1500, 88), rectAtX(1050)),
       pos: new Vec2(1090, 734),
     },
     {
       desc: 'Multiple nodes with gap (slightly too small)',
-      nodes: map(range(500, 849, 87), rectAtX(1050)),
+      nodes: iter.map(iter.range(500, 849, 87), rectAtX(1050)),
       pos: new Vec2(1090, 892),
     },
     {
       desc: 'Multiple nodes with smallest gap',
-      nodes: chain(
-        map(range(500, 901, 20), rectAtX(1050)),
-        map(range(988, 1489, 20), rectAtX(1050)),
+      nodes: iter.chain(
+        iter.map(iter.range(500, 901, 20), rectAtX(1050)),
+        iter.map(iter.range(988, 1489, 20), rectAtX(1050)),
       ),
       pos: new Vec2(1090, 944),
     },
     {
       desc: 'Multiple nodes with smallest gap (reverse)',
-      nodes: chain(
-        map(range(1488, 987, -20), rectAtX(1050)),
-        map(range(900, 499, -20), rectAtX(1050)),
+      nodes: iter.chain(
+        iter.map(iter.range(1488, 987, -20), rectAtX(1050)),
+        iter.map(iter.range(900, 499, -20), rectAtX(1050)),
       ),
       pos: new Vec2(1090, 944),
     },
     {
       desc: 'Multiple nodes with gap that is too small',
-      nodes: chain(
-        map(range(500, 901, 20), rectAtX(1050)),
-        map(range(987, 1488, 20), rectAtX(1050)),
+      nodes: iter.chain(
+        iter.map(iter.range(500, 901, 20), rectAtX(1050)),
+        iter.map(iter.range(987, 1488, 20), rectAtX(1050)),
       ),
       // This gap is 1px smaller than the previous test - so, 1px too small.
       // This position is offscreen (y >= 1000), so we pan so that the new node is centered (1531 - 690).
@@ -139,9 +138,9 @@ describe('Non dictated placement', () => {
     },
     {
       desc: 'Multiple nodes with gap that is too small (each range reversed)',
-      nodes: chain(
-        map(range(900, 499, -20), rectAtX(1050)),
-        map(range(1487, 986, -20), rectAtX(1050)),
+      nodes: iter.chain(
+        iter.map(iter.range(900, 499, -20), rectAtX(1050)),
+        iter.map(iter.range(1487, 986, -20), rectAtX(1050)),
       ),
       pos: new Vec2(1090, 1531),
       pan: new Vec2(0, 841),
@@ -259,64 +258,64 @@ describe('Previous node dictated placement', () => {
     // === Multiple node tests ===
     {
       desc: 'Multiple nodes',
-      nodes: map(range(1000, 2001, 100), rectAtY(734)),
+      nodes: iter.map(iter.range(1000, 2001, 100), rectAtY(734)),
       pos: new Vec2(2124, 734),
       pan: new Vec2(1034, 44),
     },
     {
       desc: 'Multiple nodes (reverse)',
-      nodes: map(range(2000, 999, -100), rectAtY(734)),
+      nodes: iter.map(iter.range(2000, 999, -100), rectAtY(734)),
       pos: new Vec2(2124, 734),
       pan: new Vec2(1034, 44),
     },
     {
       desc: 'Multiple nodes with gap',
-      nodes: chain(
-        map(range(1000, 1401, 100), rectAtY(734)),
-        map(range(1700, 2001, 100), rectAtY(734)),
+      nodes: iter.chain(
+        iter.map(iter.range(1000, 1401, 100), rectAtY(734)),
+        iter.map(iter.range(1700, 2001, 100), rectAtY(734)),
       ),
       pos: new Vec2(1524, 734),
     },
     {
       desc: 'Multiple nodes with gap (just big enough)',
-      nodes: map(range(1050, 2000, 248), rectAtY(734)),
+      nodes: iter.map(iter.range(1050, 2000, 248), rectAtY(734)),
       pos: new Vec2(1174, 734),
     },
     {
       desc: 'Multiple nodes with gap (slightly too small)',
-      nodes: map(range(1050, 1792, 247), rectAtY(734)),
+      nodes: iter.map(iter.range(1050, 1792, 247), rectAtY(734)),
       pos: new Vec2(1915, 734),
     },
     {
       desc: 'Multiple nodes with smallest gap',
-      nodes: chain(
-        map(range(1000, 1401, 100), rectAtY(734)),
-        map(range(1648, 1949, 100), rectAtY(734)),
+      nodes: iter.chain(
+        iter.map(iter.range(1000, 1401, 100), rectAtY(734)),
+        iter.map(iter.range(1648, 1949, 100), rectAtY(734)),
       ),
       pos: new Vec2(1524, 734),
     },
     {
       desc: 'Multiple nodes with smallest gap (reverse)',
-      nodes: chain(
-        map(range(1948, 1647, -100), rectAtY(734)),
-        map(range(1400, 999, -100), rectAtY(734)),
+      nodes: iter.chain(
+        iter.map(iter.range(1948, 1647, -100), rectAtY(734)),
+        iter.map(iter.range(1400, 999, -100), rectAtY(734)),
       ),
       pos: new Vec2(1524, 734),
     },
     {
       desc: 'Multiple nodes with gap that is too small',
-      nodes: chain(
-        map(range(1000, 1401, 100), rectAtY(734)),
-        map(range(1647, 1948, 100), rectAtY(734)),
+      nodes: iter.chain(
+        iter.map(iter.range(1000, 1401, 100), rectAtY(734)),
+        iter.map(iter.range(1647, 1948, 100), rectAtY(734)),
       ),
       pos: new Vec2(2071, 734),
       pan: new Vec2(981, 44),
     },
     {
       desc: 'Multiple nodes with gap that is too small (each range reversed)',
-      nodes: chain(
-        map(range(1400, 999, -100), rectAtY(734)),
-        map(range(1947, 1646, -100), rectAtY(734)),
+      nodes: iter.chain(
+        iter.map(iter.range(1400, 999, -100), rectAtY(734)),
+        iter.map(iter.range(1947, 1646, -100), rectAtY(734)),
       ),
       pos: new Vec2(2071, 734),
       pan: new Vec2(981, 44),

app/gui/src/project-view/components/ComponentBrowser/component.ts

@@ -10,7 +10,8 @@ import { isSome } from '@/util/data/opt'
 import { Range } from '@/util/data/range'
 import { displayedIconOf } from '@/util/getIconName'
 import type { Icon } from '@/util/iconName'
-import { qnLastSegmentIndex, QualifiedName, tryQualifiedName } from '@/util/qualifiedName'
+import type { QualifiedName } from '@/util/qualifiedName'
+import { qnLastSegmentIndex, tryQualifiedName } from '@/util/qualifiedName'
 import { unwrap } from 'ydoc-shared/util/data/result'
 
 interface ComponentLabelInfo {

app/gui/src/project-view/components/ComponentBrowser/scrolling.ts

@@ -1,5 +1,5 @@
 import { useApproach } from '@/composables/animation'
-import { ToValue } from '@/util/reactivity'
+import type { ToValue } from '@/util/reactivity'
 import { computed, ref, toValue } from 'vue'
 
 export type ScrollTarget =

app/gui/src/project-view/components/DockPanel.vue

@@ -29,9 +29,13 @@ defineExpose({ root })
 const computedSize = useResizeObserver(slideInPanel)
 const computedBounds = computed(() => new Rect(Vec2.Zero, computedSize.value))
 
-const style = computed(() => ({
-  '--dock-panel-width': size.value != null ? `${size.value}px` : 'var(--right-dock-default-width)',
-}))
+const style = computed(() =>
+  size.value != null ?
+    {
+      '--dock-panel-width': `${size.value}px`,
+    }
+  : undefined,
+)
 
 const tabStyle = {
   clipPath: tabClipPath(TAB_SIZE_PX, TAB_RADIUS_PX, 'right'),
@@ -52,36 +56,36 @@ const tabStyle = {
       :class="{ aboveFullscreen: contentFullscreen }"
     />
     <SizeTransition width :duration="100">
-      <div
-        v-if="show"
-        ref="slideInPanel"
-        :style="style"
-        class="DockPanelContent"
-        data-testid="rightDock"
-      >
-        <div class="content">
-          <slot v-if="tab == 'docs'" name="docs" />
-          <slot v-else-if="tab == 'help'" name="help" />
-        </div>
-        <div class="tabBar">
-          <div class="tab" :style="tabStyle">
-            <ToggleIcon
-              :modelValue="tab == 'docs'"
-              title="Documentation Editor"
-              icon="text"
-              @update:modelValue="tab = 'docs'"
-            />
+      <div v-if="show" ref="slideInPanel" :style="style" class="panelOuter" data-testid="rightDock">
+        <div class="panelInner">
+          <div class="content">
+            <slot v-if="tab == 'docs'" name="docs" />
+            <slot v-else-if="tab == 'help'" name="help" />
           </div>
-          <div class="tab" :style="tabStyle">
-            <ToggleIcon
-              :modelValue="tab == 'help'"
-              title="Component Help"
-              icon="help"
-              @update:modelValue="tab = 'help'"
-            />
+          <div class="tabBar">
+            <div class="tab" :style="tabStyle">
+              <ToggleIcon
+                :modelValue="tab == 'docs'"
+                title="Documentation Editor"
+                icon="text"
+                @update:modelValue="tab = 'docs'"
+              />
+            </div>
+            <div class="tab" :style="tabStyle">
+              <ToggleIcon
+                :modelValue="tab == 'help'"
+                title="Component Help"
+                icon="help"
+                @update:modelValue="tab = 'help'"
+              />
+            </div>
           </div>
+          <ResizeHandles
+            left
+            :modelValue="computedBounds"
+            @update:modelValue="size = $event.width"
+          />
         </div>
-        <ResizeHandles left :modelValue="computedBounds" @update:modelValue="size = $event.width" />
       </div>
     </SizeTransition>
   </div>
@@ -89,12 +93,25 @@ const tabStyle = {
 
 <style scoped>
 .DockPanel {
-  display: contents;
+  display: block;
+  --dock-panel-min-width: 258px;
+  width: fit-content;
 }
 
-.DockPanelContent {
-  min-width: 258px;
-  width: var(--dock-panel-width);
+/* Outer panel container; this element's visible width will be overwritten by the size transition, but the inner panel's
+ * will not, preventing content reflow. Content reflow is disruptive to the appearance of the transition, and can affect
+ * the framerate drastically.
+ */
+.panelOuter {
+  min-width: var(--dock-panel-min-width);
+  width: var(--dock-panel-width, var(--right-dock-default-width));
+  height: 100%;
+}
+
+.panelInner {
+  min-width: var(--dock-panel-min-width);
+  width: var(--dock-panel-width, var(--right-dock-default-width));
+  height: 100%;
   position: relative;
   --icon-margin: 16px; /* `--icon-margin` in `.toggleDock` must match this value. */
   --icon-size: 16px;
@@ -105,7 +122,6 @@ const tabStyle = {
 
 .content {
   width: 100%;
-  height: 100%;
   background-color: #fff;
   min-width: 0;
 }

app/gui/src/project-view/components/DocumentationEditor.vue

@@ -9,8 +9,11 @@ import type { ToValue } from '@/util/reactivity'
 import { ref, toRef, toValue, watch } from 'vue'
 import type { Path } from 'ydoc-shared/languageServerTypes'
 import { Err, Ok, mapOk, withContext, type Result } from 'ydoc-shared/util/data/result'
+import * as Y from 'yjs'
 
-const documentation = defineModel<string>({ required: true })
+const { yText } = defineProps<{
+  yText: Y.Text
+}>()
 const emit = defineEmits<{
   'update:fullscreen': [boolean]
 }>()
@@ -88,7 +91,7 @@ watch(
       </div>
       <div class="scrollArea">
         <MarkdownEditor
-          v-model="documentation"
+          :yText="yText"
           :transformImageUrl="transformImageUrl"
           :toolbarContainer="toolbarElement"
         />

app/gui/src/project-view/components/DocumentationPanel/ir.ts

@@ -3,6 +3,7 @@ import type { SuggestionEntry, SuggestionId } from '@/stores/suggestionDatabase/
 import { SuggestionKind, entryQn } from '@/stores/suggestionDatabase/entry'
 import type { Doc } from '@/util/docParser'
 import type { QualifiedName } from '@/util/qualifiedName'
+import * as iter from 'enso-common/src/utilities/data/iter'
 import type { SuggestionEntryArgument } from 'ydoc-shared/languageServerTypes/suggestions'
 
 // === Types ===
@@ -108,15 +109,15 @@ export function lookupDocumentation(db: SuggestionDb, id: SuggestionId): Docs {
 
 function getChildren(db: SuggestionDb, id: SuggestionId, kind: SuggestionKind): Docs[] {
   if (!id) return []
-  const children = Array.from(db.childIdToParentId.reverseLookup(id))
-  return children.reduce((acc: Docs[], id: SuggestionId) => {
-    const entry = db.get(id)
-    if (entry?.kind === kind && !entry?.isPrivate) {
-      const docs = lookupDocumentation(db, id)
-      acc.push(docs)
-    }
-    return acc
-  }, [])
+  const children = db.childIdToParentId.reverseLookup(id)
+  return [
+    ...iter.filterDefined(
+      iter.map(children, (id: SuggestionId) => {
+        const entry = db.get(id)
+        return entry?.kind === kind && !entry?.isPrivate ? lookupDocumentation(db, id) : undefined
+      }),
+    ),
+  ]
 }
 
 function asFunctionDocs(docs: Docs[]): FunctionDocs[] {

app/gui/src/project-view/components/EditorRoot.vue

@@ -0,0 +1,29 @@
+<script setup lang="ts">
+import { ref } from 'vue'
+
+const rootElement = ref<HTMLElement>()
+defineExpose({ rootElement })
+</script>
+
+<template>
+  <div
+    ref="rootElement"
+    class="EditorRoot"
+    @keydown.arrow-left.stop
+    @keydown.arrow-right.stop
+    @keydown.arrow-up.stop
+    @keydown.arrow-down.stop
+    @keydown.enter.stop
+    @keydown.backspace.stop
+    @keydown.delete.stop
+    @wheel.stop.passive
+    @contextmenu.stop
+  ></div>
+</template>
+
+<style scoped>
+.EditorRoot {
+  width: 100%;
+  height: 100%;
+}
+</style>

app/gui/src/project-view/components/GraphEditor.vue

@@ -9,7 +9,7 @@ import {
 import BottomPanel from '@/components/BottomPanel.vue'
 import CodeEditor from '@/components/CodeEditor.vue'
 import ComponentBrowser from '@/components/ComponentBrowser.vue'
-import { type Usage } from '@/components/ComponentBrowser/input'
+import type { Usage } from '@/components/ComponentBrowser/input'
 import { usePlacement } from '@/components/ComponentBrowser/placement'
 import ComponentDocumentation from '@/components/ComponentDocumentation.vue'
 import DockPanel from '@/components/DockPanel.vue'
@@ -20,21 +20,21 @@ import { useGraphEditorClipboard } from '@/components/GraphEditor/clipboard'
 import { performCollapse, prepareCollapsedInfo } from '@/components/GraphEditor/collapsing'
 import type { NodeCreationOptions } from '@/components/GraphEditor/nodeCreation'
 import { useGraphEditorToasts } from '@/components/GraphEditor/toasts'
-import { Uploader, uploadedExpression } from '@/components/GraphEditor/upload'
+import { uploadedExpression, Uploader } from '@/components/GraphEditor/upload'
 import GraphMissingView from '@/components/GraphMissingView.vue'
 import GraphMouse from '@/components/GraphMouse.vue'
 import PlusButton from '@/components/PlusButton.vue'
 import SceneScroller from '@/components/SceneScroller.vue'
 import TopBar from '@/components/TopBar.vue'
 import { builtinWidgets } from '@/components/widgets'
-import { useAstDocumentation } from '@/composables/astDocumentation'
 import { useDoubleClick } from '@/composables/doubleClick'
 import { keyboardBusy, keyboardBusyExceptIn, unrefElement, useEvent } from '@/composables/events'
 import { groupColorVar } from '@/composables/nodeColors'
 import type { PlacementStrategy } from '@/composables/nodeCreation'
 import { useSyncLocalStorage } from '@/composables/syncLocalStorage'
 import { provideFullscreenContext } from '@/providers/fullscreenContext'
-import { provideGraphNavigator, type GraphNavigator } from '@/providers/graphNavigator'
+import type { GraphNavigator } from '@/providers/graphNavigator'
+import { provideGraphNavigator } from '@/providers/graphNavigator'
 import { provideNodeColors } from '@/providers/graphNodeColors'
 import { provideNodeCreation } from '@/providers/graphNodeCreation'
 import { provideGraphSelection } from '@/providers/graphSelection'
@@ -43,25 +43,25 @@ import { provideInteractionHandler } from '@/providers/interactionHandler'
 import { provideKeyboard } from '@/providers/keyboard'
 import { injectVisibility } from '@/providers/visibility'
 import { provideWidgetRegistry } from '@/providers/widgetRegistry'
-import { provideGraphStore, type NodeId } from '@/stores/graph'
+import type { NodeId } from '@/stores/graph'
+import { provideGraphStore } from '@/stores/graph'
 import type { RequiredImport } from '@/stores/graph/imports'
 import { useProjectStore } from '@/stores/project'
 import { useSettings } from '@/stores/settings'
 import { provideSuggestionDbStore } from '@/stores/suggestionDatabase'
-import type { SuggestionId } from '@/stores/suggestionDatabase/entry'
-import { suggestionDocumentationUrl, type Typename } from '@/stores/suggestionDatabase/entry'
+import type { SuggestionId, Typename } from '@/stores/suggestionDatabase/entry'
+import { suggestionDocumentationUrl } from '@/stores/suggestionDatabase/entry'
 import { provideVisualizationStore } from '@/stores/visualization'
 import { bail } from '@/util/assert'
 import { Ast } from '@/util/ast'
-import type { AstId } from '@/util/ast/abstract'
 import { colorFromString } from '@/util/colors'
 import { partition } from '@/util/data/array'
-import { every, filterDefined } from '@/util/data/iterable'
 import { Rect } from '@/util/data/rect'
-import { Err, Ok, unwrapOr } from '@/util/data/result'
+import { Err, Ok } from '@/util/data/result'
 import { Vec2 } from '@/util/data/vec2'
 import { computedFallback, useSelectRef } from '@/util/reactivity'
 import { until } from '@vueuse/core'
+import * as iter from 'enso-common/src/utilities/data/iter'
 import { encoding, set } from 'lib0'
 import {
   computed,
@@ -75,7 +75,6 @@ import {
   type ComponentInstance,
 } from 'vue'
 import { encodeMethodPointer } from 'ydoc-shared/languageServerTypes'
-import * as iterable from 'ydoc-shared/util/data/iterable'
 import { isDevMode } from 'ydoc-shared/util/detect'
 
 const rootNode = ref<HTMLElement>()
@@ -330,7 +329,7 @@ const graphBindingsHandler = graphBindings.handler({
   },
   toggleVisualization() {
     const selected = nodeSelection.selected
-    const allVisible = every(
+    const allVisible = iter.every(
       selected,
       (id) => graphStore.db.nodeIdToNode.get(id)?.vis?.visible === true,
     )
@@ -416,7 +415,7 @@ const documentationEditorArea = computed(() => unrefElement(docEditor))
 const showRightDock = computedFallback(
   storedShowRightDock,
   // Show documentation editor when documentation exists on first graph visit.
-  () => !!documentation.state.value,
+  () => (markdownDocs.value?.length ?? 0) > 0,
 )
 const rightDockTab = computedFallback(storedRightDockTab, () => 'docs')
 
@@ -430,9 +429,11 @@ const documentationEditorHandler = documentationEditorBindings.handler({
   },
 })
 
-const { documentation } = useAstDocumentation(graphStore, () =>
-  unwrapOr(graphStore.methodAst, undefined),
-)
+const markdownDocs = computed(() => {
+  const currentMethod = graphStore.methodAst
+  if (!currentMethod.ok) return
+  return currentMethod.value.mutableDocumentationMarkdown()
+})
 
 // === Component Browser ===
 
@@ -550,7 +551,7 @@ const componentBrowserElements = computed(() => [
 
 interface NewNodeOptions {
   placement: PlacementStrategy
-  sourcePort?: AstId | undefined
+  sourcePort?: Ast.AstId | undefined
 }
 
 function addNodeDisconnected() {
@@ -592,7 +593,7 @@ function createNodesFromSource(sourceNode: NodeId, options: NodeCreationOptions[
     createWithComponentBrowser({ placement: { type: 'source', node: sourceNode }, sourcePort })
 }
 
-function handleNodeOutputPortDoubleClick(id: AstId) {
+function handleNodeOutputPortDoubleClick(id: Ast.AstId) {
   const srcNode = graphStore.db.getPatternExpressionNodeId(id)
   if (srcNode == null) {
     console.error('Impossible happened: Double click on port not belonging to any node: ', id)
@@ -601,7 +602,7 @@ function handleNodeOutputPortDoubleClick(id: AstId) {
   createWithComponentBrowser({ placement: { type: 'source', node: srcNode }, sourcePort: id })
 }
 
-function handleEdgeDrop(source: AstId, position: Vec2) {
+function handleEdgeDrop(source: Ast.AstId, position: Vec2) {
   createWithComponentBrowser({ placement: { type: 'fixed', position }, sourcePort: source })
 }
 
@@ -609,7 +610,7 @@ function handleEdgeDrop(source: AstId, position: Vec2) {
 
 function collapseNodes() {
   const selected = new Set(
-    iterable.filter(
+    iter.filter(
       nodeSelection.selected,
       (id) => graphStore.db.nodeIdToNode.get(id)?.type === 'component',
     ),
@@ -630,7 +631,7 @@ function collapseNodes() {
     if (!topLevel) {
       bail('BUG: no top level, collapsing not possible.')
     }
-    const selectedNodeRects = filterDefined(Array.from(selected, graphStore.visibleArea))
+    const selectedNodeRects = iter.filterDefined(iter.map(selected, graphStore.visibleArea))
     graphStore.edit((edit) => {
       const { collapsedCallRoot, collapsedNodeIds, outputAstId } = performCollapse(
         info.value,
@@ -641,8 +642,8 @@ function collapseNodes() {
       const position = collapsedNodePlacement(selectedNodeRects)
       edit.get(collapsedCallRoot).mutableNodeMetadata().set('position', position.xy())
       if (outputAstId != null) {
-        const collapsedNodeRects = filterDefined(
-          Array.from(collapsedNodeIds, graphStore.visibleArea),
+        const collapsedNodeRects = iter.filterDefined(
+          iter.map(collapsedNodeIds, graphStore.visibleArea),
         )
         const { place } = usePlacement(collapsedNodeRects, graphNavigator.viewport)
         const position = place(collapsedNodeRects)
@@ -785,9 +786,9 @@ const documentationEditorFullscreen = ref(false)
     >
       <template #docs>
         <DocumentationEditor
+          v-if="markdownDocs"
           ref="docEditor"
-          :modelValue="documentation.state.value"
-          @update:modelValue="documentation.set"
+          :yText="markdownDocs"
           @update:fullscreen="documentationEditorFullscreen = $event"
         />
       </template>
@@ -812,7 +813,7 @@ const documentationEditorFullscreen = ref(false)
 
   display: flex;
   flex-direction: row;
-  & :deep(.DockPanel) {
+  & .DockPanel {
     flex: none;
   }
   & .vertical {
@@ -824,7 +825,7 @@ const documentationEditorFullscreen = ref(false)
 .vertical {
   display: flex;
   flex-direction: column;
-  & :deep(.BottomPanel) {
+  & .BottomPanel {
     flex: none;
   }
   & .viewport {

app/gui/src/project-view/components/GraphEditor/GraphVisualization.vue

@@ -1,15 +1,13 @@
 <script setup lang="ts">
 import { visualizationBindings } from '@/bindings'
-import {
-  RawDataSource,
-  useVisualizationData,
-} from '@/components/GraphEditor/GraphVisualization/visualizationData'
+import type { RawDataSource } from '@/components/GraphEditor/GraphVisualization/visualizationData'
+import { useVisualizationData } from '@/components/GraphEditor/GraphVisualization/visualizationData'
 import VisualizationToolbar from '@/components/GraphEditor/GraphVisualization/VisualizationToolbar.vue'
 import type { NodeCreationOptions } from '@/components/GraphEditor/nodeCreation'
 import ResizeHandles from '@/components/ResizeHandles.vue'
 import WithFullscreenMode from '@/components/WithFullscreenMode.vue'
 import { focusIsIn, useEvent, useResizeObserver } from '@/composables/events'
-import { VisualizationDataSource } from '@/stores/visualization'
+import type { VisualizationDataSource } from '@/stores/visualization'
 import type { Opt } from '@/util/data/opt'
 import { type BoundsSet, Rect } from '@/util/data/rect'
 import { Vec2 } from '@/util/data/vec2'

app/gui/src/project-view/components/GraphEditor/GraphVisualization/VisualizationToolbar.vue

@@ -3,18 +3,18 @@ import FullscreenButton from '@/components/FullscreenButton.vue'
 import SelectionDropdown from '@/components/SelectionDropdown.vue'
 import SvgButton from '@/components/SvgButton.vue'
 import ToggleIcon from '@/components/ToggleIcon.vue'
+import type { ToolbarItem } from '@/components/visualizations/toolbar'
 import {
   isActionButton,
   isSelectionMenu,
   isToggleButton,
-  ToolbarItem,
 } from '@/components/visualizations/toolbar'
 import VisualizationSelector from '@/components/VisualizationSelector.vue'
 import { useEvent } from '@/composables/events'
 import { provideInteractionHandler } from '@/providers/interactionHandler'
 import { isQualifiedName, qnLastSegment } from '@/util/qualifiedName'
 import { computed, toValue } from 'vue'
-import { VisualizationIdentifier } from 'ydoc-shared/yjsModel'
+import type { VisualizationIdentifier } from 'ydoc-shared/yjsModel'
 
 const isFullscreen = defineModel<boolean>('isFullscreen', { required: true })
 const currentVis = defineModel<VisualizationIdentifier>('currentVis', { required: true })

app/gui/src/project-view/components/GraphEditor/GraphVisualization/visualizationData.ts

@@ -1,25 +1,25 @@
 import LoadingErrorVisualization from '@/components/visualizations/LoadingErrorVisualization.vue'
 import LoadingVisualization from '@/components/visualizations/LoadingVisualization.vue'
-import { ToolbarItem } from '@/components/visualizations/toolbar'
+import type { ToolbarItem } from '@/components/visualizations/toolbar'
 import { useProjectStore } from '@/stores/project'
 import type { NodeVisualizationConfiguration } from '@/stores/project/executionContext'
 import {
   DEFAULT_VISUALIZATION_CONFIGURATION,
   DEFAULT_VISUALIZATION_IDENTIFIER,
   useVisualizationStore,
-  VisualizationDataSource,
+  type VisualizationDataSource,
 } from '@/stores/visualization'
 import type { Visualization } from '@/stores/visualization/runtimeTypes'
 import { Ast } from '@/util/ast'
 import { toError } from '@/util/data/error'
-import { ToValue } from '@/util/reactivity'
+import type { ToValue } from '@/util/reactivity'
 import { computedAsync } from '@vueuse/core'
 import {
   computed,
   onErrorCaptured,
   ref,
   shallowRef,
-  ShallowRef,
+  type ShallowRef,
   toValue,
   watch,
   watchEffect,

app/gui/src/project-view/components/GraphEditor/__tests__/collapsing.test.ts

@@ -15,10 +15,10 @@ import { nodeIdFromOuterAst } from '../../../stores/graph/graphDatabase'
 // ===============================
 
 function setupGraphDb(code: string, graphDb: GraphDb) {
-  const { root, toRaw, getSpan } = Ast.parseExtended(code)
+  const { root, toRaw, getSpan } = Ast.parseUpdatingIdMap(code)
   const expressions = Array.from(root.statements())
   const func = expressions[0]
-  assert(func instanceof Ast.Function)
+  assert(func instanceof Ast.FunctionDef)
   const rawFunc = toRaw.get(func.id)
   assert(rawFunc?.type === RawAst.Tree.Type.Function)
   graphDb.updateExternalIds(root)

app/gui/src/project-view/components/GraphEditor/clipboard.ts

@@ -2,9 +2,9 @@ import type { NodeCreationOptions } from '@/composables/nodeCreation'
 import type { GraphStore, Node, NodeId } from '@/stores/graph'
 import { Ast } from '@/util/ast'
 import { Pattern } from '@/util/ast/match'
-import { filterDefined } from '@/util/data/iterable'
 import { Vec2 } from '@/util/data/vec2'
 import type { ToValue } from '@/util/reactivity'
+import * as iter from 'enso-common/src/utilities/data/iter'
 import { computed, toValue } from 'vue'
 import type { NodeMetadataFields } from 'ydoc-shared/ast'
 
@@ -131,7 +131,7 @@ async function decodeClipboard<T>(
       }
     }
   }
-  return filterDefined(await Promise.all(clipboardItems.map(decodeItem)))
+  return iter.filterDefined(await Promise.all(clipboardItems.map(decodeItem)))
 }
 
 // === Spreadsheet clipboard decoder ===

app/gui/src/project-view/components/GraphEditor/collapsing.ts

@@ -1,8 +1,10 @@
-import { GraphDb, NodeId, nodeIdFromOuterAst } from '@/stores/graph/graphDatabase'
+import type { GraphDb, NodeId } from '@/stores/graph/graphDatabase'
+import { nodeIdFromOuterAst } from '@/stores/graph/graphDatabase'
 import { assert } from '@/util/assert'
 import { Ast } from '@/util/ast'
-import { Identifier, isIdentifier, moduleMethodNames } from '@/util/ast/abstract'
-import { Err, Ok, Result, unwrap } from '@/util/data/result'
+import type { Identifier } from '@/util/ast/abstract'
+import { isIdentifier, moduleMethodNames } from '@/util/ast/abstract'
+import { Err, Ok, unwrap, type Result } from '@/util/data/result'
 import { tryIdentifier } from '@/util/qualifiedName'
 import * as set from 'lib0/set'
 
@@ -222,7 +224,7 @@ export function performCollapseImpl(
   const collapsedBody = Ast.BodyBlock.new(extractedLines, edit)
   const outputAst = Ast.Ident.new(edit, outputIdentifier)
   collapsedBody.push(outputAst)
-  const collapsedFunction = Ast.Function.new(collapsedName, info.args, collapsedBody, {
+  const collapsedFunction = Ast.FunctionDef.new(collapsedName, info.args, collapsedBody, {
     edit,
     documentation: 'ICON group',
   })

app/gui/src/project-view/components/GraphEditor/dragging.ts

@@ -6,7 +6,7 @@ import type { Opt } from '@/util/data/opt'
 import { Rect } from '@/util/data/rect'
 import { Vec2 } from '@/util/data/vec2'
 import theme from '@/util/theme.json'
-import { iteratorFilter } from 'lib0/iterator'
+import * as iter from 'enso-common/src/utilities/data/iter'
 import { computed, markRaw, ref, watchEffect, type ComputedRef, type WatchStopHandle } from 'vue'
 
 const DRAG_SNAP_THRESHOLD = 16
@@ -21,7 +21,7 @@ interface PartialVec2 {
  * Snap Grid for dragged nodes.
  *
  * Created from existing nodes' rects, it allows "snapping" dragged nodes to another nodes on
- * the scene, so the user could easily and nicely ailgn their nodes.
+ * the scene, so the user could easily and nicely align their nodes.
  *
  * The nodes will be snapped to align with every edge of any other node, and also at place above
  * and below node leaving default vertical gap (same as when adding new node).
@@ -199,7 +199,7 @@ export function useDragging() {
 
     private createSnapGrid() {
       const nonDraggedRects = computed(() => {
-        const nonDraggedNodes = iteratorFilter(
+        const nonDraggedNodes = iter.filter(
           graphStore.db.nodeIds(),
           (id) => !this.draggedNodes.has(id),
         )

app/gui/src/project-view/components/GraphEditor/widgets/WidgetTableEditor/TableHeader.vue

@@ -1,6 +1,7 @@
 <script lang="ts">
 import SvgButton from '@/components/SvgButton.vue'
-import { provideTooltipRegistry, TooltipRegistry } from '@/providers/tooltipState'
+import type { TooltipRegistry } from '@/providers/tooltipState'
+import { provideTooltipRegistry } from '@/providers/tooltipState'
 import type { IHeaderParams } from 'ag-grid-community'
 import { computed, ref, watch } from 'vue'
 

app/gui/src/project-view/components/GraphEditor/widgets/WidgetTableEditor/tableNewArgument.ts

@@ -1,18 +1,18 @@
 import { commonContextMenuActions, type MenuItem } from '@/components/shared/AgGridTableView.vue'
 import type { WidgetInput, WidgetUpdate } from '@/providers/widgetRegistry'
-import { requiredImportsByFQN, type RequiredImport } from '@/stores/graph/imports'
+import { type RequiredImport, requiredImportsByFQN } from '@/stores/graph/imports'
 import type { SuggestionDb } from '@/stores/suggestionDatabase'
 import { assert } from '@/util/assert'
 import { Ast } from '@/util/ast'
 import { tryEnsoToNumber, tryNumberToEnso } from '@/util/ast/abstract'
 import { findIndexOpt } from '@/util/data/array'
-import * as iterable from '@/util/data/iterable'
-import { Err, Ok, transposeResult, unwrapOrWithLog, type Result } from '@/util/data/result'
+import { Err, Ok, type Result, transposeResult, unwrapOrWithLog } from '@/util/data/result'
 import { qnLastSegment, type QualifiedName } from '@/util/qualifiedName'
 import type { ToValue } from '@/util/reactivity'
 import type { ColDef } from 'ag-grid-enterprise'
+import * as iter from 'enso-common/src/utilities/data/iter'
 import { computed, toValue } from 'vue'
-import { ColumnSpecificHeaderParams } from './TableHeader.vue'
+import type { ColumnSpecificHeaderParams } from './TableHeader.vue'
 
 /** Id of a fake column with "Add new column" option. */
 export const NEW_COLUMN_ID = 'NewColumn'
@@ -437,7 +437,7 @@ export function useTableNewArgument(
     }
     const edit = graph.startEdit()
     const columns = edit.getVersion(columnsAst.value.value)
-    const fromIndex = iterable.find(columns.enumerate(), ([, ast]) => ast?.id === colId)?.[0]
+    const fromIndex = iter.find(columns.enumerate(), ([, ast]) => ast?.id === colId)?.[0]
     if (fromIndex != null) {
       columns.move(fromIndex, toIndex - 1)
       onUpdate({ edit })

app/gui/src/project-view/components/MarkdownEditor.vue

@@ -1,9 +1,10 @@
 <script setup lang="ts">
 import type { UrlTransformer } from '@/components/MarkdownEditor/imageUrlTransformer'
 import { defineAsyncComponent } from 'vue'
+import * as Y from 'yjs'
 
-const text = defineModel<string>({ required: true })
 const props = defineProps<{
+  yText: Y.Text
   transformImageUrl?: UrlTransformer
   toolbarContainer: HTMLElement | undefined
 }>()
@@ -15,6 +16,6 @@ const LazyMarkdownEditor = defineAsyncComponent(
 
 <template>
   <Suspense>
-    <LazyMarkdownEditor v-model="text" v-bind="props" />
+    <LazyMarkdownEditor v-bind="props" />
   </Suspense>
 </template>

app/gui/src/project-view/components/MarkdownEditor/BlockTypeMenu.vue

@@ -1,39 +0,0 @@
-<script setup lang="ts">
-import { blockTypeToBlockName, type BlockType } from '@/components/MarkdownEditor/formatting'
-import SelectionDropdown from '@/components/SelectionDropdown.vue'
-import type { Icon } from '@/util/iconName'
-
-const blockType = defineModel<BlockType>({ required: true })
-
-const blockTypeIcon: Record<keyof typeof blockTypeToBlockName, Icon> = {
-  paragraph: 'text',
-  bullet: 'bullet-list',
-  code: 'code',
-  h1: 'header1',
-  h2: 'header2',
-  h3: 'header3',
-  number: 'numbered-list',
-  quote: 'quote',
-}
-const blockTypesOrdered: BlockType[] = [
-  'paragraph',
-  'h1',
-  'h2',
-  'h3',
-  'code',
-  'bullet',
-  'number',
-  'quote',
-]
-
-const blockTypeOptions = Object.fromEntries(
-  blockTypesOrdered.map((key) => [
-    key,
-    { icon: blockTypeIcon[key], label: blockTypeToBlockName[key] },
-  ]),
-)
-</script>
-
-<template>
-  <SelectionDropdown v-model="blockType" :options="blockTypeOptions" labelButton />
-</template>

app/gui/src/project-view/components/MarkdownEditor/DocumentationImage.vue

@@ -1,6 +1,6 @@
 <script setup lang="ts">
 import {
-  injectLexicalImageUrlTransformer,
+  injectDocumentationImageUrlTransformer,
   type TransformUrlResult,
 } from '@/components/MarkdownEditor/imageUrlTransformer'
 import { computedAsync } from '@vueuse/core'
@@ -14,10 +14,10 @@ const props = defineProps<{
   alt: string
 }>()
 
-const urlTransformer = injectLexicalImageUrlTransformer(true)
+const urlTransformer = injectDocumentationImageUrlTransformer(true)
 
-// NOTE: Garbage-collecting image data when the `src` changes is not implemented. Current users of `LexicalImage` don't
-// change the `src` after creating an image.
+// NOTE: Garbage-collecting image data when the `src` changes is not implemented. Current users of `DocumentationImage`
+// don't change the `src` after creating an image.
 const data: Ref<TransformUrlResult | undefined> =
   urlTransformer ?
     computedAsync(() => urlTransformer.transformUrl(props.src), undefined, {

app/gui/src/project-view/components/MarkdownEditor/FormatPropertiesBar.vue

@@ -1,87 +0,0 @@
-<script setup lang="ts">
-import DropdownMenu from '@/components/DropdownMenu.vue'
-import { type UseFormatting } from '@/components/MarkdownEditor/formatting'
-import SvgButton from '@/components/SvgButton.vue'
-import ToggleIcon from '@/components/ToggleIcon.vue'
-import type { Icon } from '@/util/iconName'
-import { computed, ref, type Ref } from 'vue'
-
-const props = defineProps<{ formatting: UseFormatting }>()
-
-const menuOpen = ref(false)
-
-const { bold, italic, strikethrough, subscript, superscript, blockType, clearFormatting } =
-  props.formatting
-
-function useValueEqualsConstant<T>(value: Ref<T>, constant: T, valueWhenSetToFalse: T) {
-  return {
-    state: computed(() => value.value === constant),
-    set: (newValue: boolean) => {
-      if (newValue && value.value !== constant) {
-        value.value = constant
-      } else if (!newValue && value.value === constant) {
-        value.value = valueWhenSetToFalse
-      }
-    },
-  }
-}
-const code = useValueEqualsConstant(blockType.state, 'code', 'paragraph')
-
-const TODO: Icon = 'text'
-
-const close = () => (menuOpen.value = false)
-</script>
-
-<template>
-  <ToggleIcon
-    :modelValue="bold.state.value"
-    icon="bold"
-    title="Bold"
-    @update:modelValue="bold.set"
-  />
-  <ToggleIcon
-    :modelValue="italic.state.value"
-    icon="italic"
-    title="Italic"
-    @update:modelValue="italic.set"
-  />
-  <ToggleIcon
-    :modelValue="code.state.value"
-    icon="code"
-    title="Insert Code Block"
-    @update:modelValue="code.set"
-  />
-  <!-- TODO: Insert link -->
-  <DropdownMenu v-model:open="menuOpen">
-    <template #button>Aa</template>
-    <template #entries>
-      <ToggleIcon
-        :modelValue="strikethrough.state.value"
-        icon="strike-through"
-        label="Strikethrough"
-        @update:modelValue="strikethrough.set"
-        @click="close"
-      />
-      <ToggleIcon
-        :modelValue="subscript.state.value"
-        :icon="TODO"
-        label="Subscript"
-        @update:modelValue="subscript.set"
-        @click="close"
-      />
-      <ToggleIcon
-        :modelValue="superscript.state.value"
-        :icon="TODO"
-        label="Superscript"
-        @update:modelValue="superscript.set"
-        @click="close"
-      />
-      <SvgButton
-        name="remove-textstyle"
-        label="Clear Formatting"
-        @click.stop="clearFormatting"
-        @click="close"
-      />
-    </template>
-  </DropdownMenu>
-</template>

app/gui/src/project-view/components/MarkdownEditor/FormattingToolbar.vue

@@ -1,45 +0,0 @@
-<script setup lang="ts">
-import BlockTypeMenu from '@/components/MarkdownEditor/BlockTypeMenu.vue'
-import FormatPropertiesBar from '@/components/MarkdownEditor/FormatPropertiesBar.vue'
-import type { UseFormatting } from '@/components/MarkdownEditor/formatting'
-
-const props = defineProps<{ formatting: UseFormatting }>()
-
-const { blockType } = props.formatting
-</script>
-
-<template>
-  <div class="FormattingToolbar">
-    <div class="section">
-      <BlockTypeMenu :modelValue="blockType.state.value" @update:modelValue="blockType.set" />
-    </div>
-    <div class="section">
-      <FormatPropertiesBar :formatting="formatting" />
-    </div>
-  </div>
-</template>
-
-<style scoped>
-.FormattingToolbar {
-  --color-frame-bg: white;
-  display: flex;
-  height: 100%;
-  flex-direction: row;
-  align-items: center;
-  gap: 6px;
-  line-height: 16px;
-}
-
-.section {
-  display: flex;
-  background-color: white;
-  border-radius: var(--radius-full);
-  gap: 4px;
-  padding: 4px;
-}
-
-.FormattingToolbar :deep(.iconLabel) {
-  margin-left: 4px;
-  padding-right: 4px;
-}
-</style>

app/gui/src/project-view/components/MarkdownEditor/ImagePlugin/imageNode.ts

@@ -1,164 +0,0 @@
-import LexicalImage from '@/components/MarkdownEditor/ImagePlugin/LexicalImage.vue'
-import type {
-  DOMConversionMap,
-  DOMConversionOutput,
-  DOMExportOutput,
-  EditorConfig,
-  LexicalNode,
-  NodeKey,
-  SerializedLexicalNode,
-  Spread,
-} from 'lexical'
-import { $applyNodeReplacement, DecoratorNode } from 'lexical'
-import { h, type Component } from 'vue'
-
-export interface ImagePayload {
-  altText: string
-  key?: NodeKey | undefined
-  src: string
-}
-
-export interface UpdateImagePayload {
-  altText?: string
-}
-
-function $convertImageElement(domNode: Node): null | DOMConversionOutput {
-  if (domNode instanceof HTMLImageElement) {
-    const { alt: altText, src } = domNode
-    const node = $createImageNode({ altText, src })
-    return { node }
-  }
-  return null
-}
-
-export type SerializedImageNode = Spread<
-  {
-    altText: string
-    src: string
-  },
-  SerializedLexicalNode
->
-
-/** TODO: Add docs */
-export class ImageNode extends DecoratorNode<Component> {
-  __src: string
-  __altText: string
-
-  /** TODO: Add docs */
-  static override getType(): string {
-    return 'image'
-  }
-
-  /** TODO: Add docs */
-  static override clone(node: ImageNode): ImageNode {
-    return new ImageNode(node.__src, node.__altText, node.__key)
-  }
-
-  /** TODO: Add docs */
-  static override importJSON(serializedNode: SerializedImageNode): ImageNode {
-    const { altText, src } = serializedNode
-    return $createImageNode({
-      altText,
-      src,
-    })
-  }
-
-  /** TODO: Add docs */
-  static override importDOM(): DOMConversionMap | null {
-    return {
-      img: (_node: Node) => ({
-        conversion: $convertImageElement,
-        priority: 0,
-      }),
-    }
-  }
-
-  /** TODO: Add docs */
-  constructor(src: string, altText: string, key?: NodeKey) {
-    super(key)
-    this.__src = src
-    this.__altText = altText
-  }
-
-  /** TODO: Add docs */
-  override exportDOM(): DOMExportOutput {
-    const element = document.createElement('img')
-    element.setAttribute('src', this.__src)
-    element.setAttribute('alt', this.__altText)
-    return { element }
-  }
-
-  /** TODO: Add docs */
-  override exportJSON(): SerializedImageNode {
-    return {
-      altText: this.getAltText(),
-      src: this.getSrc(),
-      type: 'image',
-      version: 1,
-    }
-  }
-
-  /** TODO: Add docs */
-  getSrc(): string {
-    return this.__src
-  }
-
-  /** TODO: Add docs */
-  getAltText(): string {
-    return this.__altText
-  }
-
-  /** TODO: Add docs */
-  setAltText(altText: string): void {
-    const writable = this.getWritable()
-    writable.__altText = altText
-  }
-
-  /** TODO: Add docs */
-  update(payload: UpdateImagePayload): void {
-    const writable = this.getWritable()
-    const { altText } = payload
-    if (altText !== undefined) {
-      writable.__altText = altText
-    }
-  }
-
-  // View
-
-  /** TODO: Add docs */
-  override createDOM(config: EditorConfig): HTMLElement {
-    const span = document.createElement('span')
-    const className = config.theme.image
-    if (className !== undefined) {
-      span.className = className
-    }
-    return span
-  }
-
-  /** TODO: Add docs */
-  override updateDOM(_prevNode: ImageNode, dom: HTMLElement, config: EditorConfig): false {
-    const className = config.theme.image
-    if (className !== undefined) {
-      dom.className = className
-    }
-    return false
-  }
-
-  /** TODO: Add docs */
-  override decorate(): Component {
-    return h(LexicalImage, {
-      src: this.__src,
-      alt: this.__altText,
-    })
-  }
-}
-
-/** TODO: Add docs */
-export function $createImageNode({ altText, src, key }: ImagePayload): ImageNode {
-  return $applyNodeReplacement(new ImageNode(src, altText, key))
-}
-
-/** TODO: Add docs */
-export function $isImageNode(node: LexicalNode | null | undefined): node is ImageNode {
-  return node instanceof ImageNode
-}

app/gui/src/project-view/components/MarkdownEditor/ImagePlugin/index.ts

@@ -1,36 +0,0 @@
-import {
-  $createImageNode,
-  $isImageNode,
-  ImageNode,
-} from '@/components/MarkdownEditor/ImagePlugin/imageNode'
-import type { LexicalMarkdownPlugin } from '@/components/MarkdownEditor/markdown'
-import type { TextMatchTransformer } from '@lexical/markdown'
-import type { LexicalEditor } from 'lexical'
-import { assertDefined } from 'ydoc-shared/util/assert'
-
-export const IMAGE: TextMatchTransformer = {
-  dependencies: [ImageNode],
-  export: (node) => {
-    if (!$isImageNode(node)) {
-      return null
-    }
-    return `![${node.getAltText()}](${node.getSrc()})`
-  },
-  importRegExp: /!\[([^\]]*)]\(([^()\n]+)\)/,
-  regExp: /!\[([^\]]*)]\(([^()\n]+)\)$/,
-  replace: (textNode, match) => {
-    const [, altText, src] = match
-    assertDefined(altText)
-    assertDefined(src)
-    const imageNode = $createImageNode({ altText, src })
-    textNode.replace(imageNode)
-  },
-  trigger: ')',
-  type: 'text-match',
-}
-
-export const imagePlugin: LexicalMarkdownPlugin = {
-  nodes: [ImageNode],
-  transformers: [IMAGE],
-  register(_editor: LexicalEditor): void {},
-}

app/gui/src/project-view/components/MarkdownEditor/MarkdownEditorImpl.vue

@@ -1,80 +1,154 @@
 <script setup lang="ts">
-import FloatingSelectionMenu from '@/components/FloatingSelectionMenu.vue'
-import FormattingToolbar from '@/components/MarkdownEditor/FormattingToolbar.vue'
-import { imagePlugin } from '@/components/MarkdownEditor/ImagePlugin'
-import SelectionFormattingToolbar from '@/components/MarkdownEditor/SelectionFormattingToolbar.vue'
-import { lexicalRichTextTheme, useFormatting } from '@/components/MarkdownEditor/formatting'
+import EditorRoot from '@/components/EditorRoot.vue'
+import { highlightStyle } from '@/components/MarkdownEditor/highlight'
 import {
-  provideLexicalImageUrlTransformer,
+  provideDocumentationImageUrlTransformer,
   type UrlTransformer,
 } from '@/components/MarkdownEditor/imageUrlTransformer'
-import { listPlugin } from '@/components/MarkdownEditor/listPlugin'
-import { markdownPlugin } from '@/components/MarkdownEditor/markdown'
-import { useLexical } from '@/components/lexical'
-import LexicalContent from '@/components/lexical/LexicalContent.vue'
-import LexicalDecorators from '@/components/lexical/LexicalDecorators.vue'
-import { autoLinkPlugin, linkPlugin, useLinkNode } from '@/components/lexical/LinkPlugin'
-import LinkToolbar from '@/components/lexical/LinkToolbar.vue'
-import { shallowRef, toRef, useCssModule, type ComponentInstance } from 'vue'
+import { ensoMarkdown } from '@/components/MarkdownEditor/markdown'
+import VueComponentHost from '@/components/VueComponentHost.vue'
+import { EditorState } from '@codemirror/state'
+import { EditorView } from '@codemirror/view'
+import { minimalSetup } from 'codemirror'
+import { type ComponentInstance, onMounted, ref, toRef, useCssModule, watch } from 'vue'
+import { yCollab } from 'y-codemirror.next'
+import * as awarenessProtocol from 'y-protocols/awareness.js'
+import * as Y from 'yjs'
+
+const editorRoot = ref<ComponentInstance<typeof EditorRoot>>()
 
-const markdown = defineModel<string>({ required: true })
 const props = defineProps<{
+  yText: Y.Text
   transformImageUrl?: UrlTransformer | undefined
   toolbarContainer: HTMLElement | undefined
 }>()
 
-const contentElement = shallowRef<ComponentInstance<typeof LexicalContent>>()
+const vueHost = ref<ComponentInstance<typeof VueComponentHost>>()
 
-provideLexicalImageUrlTransformer(toRef(props, 'transformImageUrl'))
+provideDocumentationImageUrlTransformer(toRef(props, 'transformImageUrl'))
 
-const theme = lexicalRichTextTheme(useCssModule('lexicalTheme'))
-const { editor } = useLexical(contentElement, 'MarkdownEditor', theme, [
-  ...markdownPlugin(markdown, [listPlugin, imagePlugin, linkPlugin]),
-  autoLinkPlugin,
-])
-const formatting = useFormatting(editor)
+const awareness = new awarenessProtocol.Awareness(new Y.Doc())
+const editorView = new EditorView()
+const constantExtensions = [minimalSetup, highlightStyle(useCssModule()), EditorView.lineWrapping]
 
-const { urlUnderCursor } = useLinkNode(editor)
+watch([vueHost, toRef(props, 'yText')], ([vueHost, yText]) => {
+  if (!vueHost) return
+  editorView.setState(
+    EditorState.create({
+      doc: yText.toString(),
+      extensions: [...constantExtensions, ensoMarkdown({ vueHost }), yCollab(yText, awareness)],
+    }),
+  )
+})
+
+onMounted(() => {
+  const content = editorView.dom.getElementsByClassName('cm-content')[0]!
+  content.addEventListener('focusin', () => (editing.value = true))
+  editorRoot.value?.rootElement?.prepend(editorView.dom)
+})
+
+const editing = ref(false)
 </script>
 
 <template>
-  <div class="MarkdownEditor fullHeight">
-    <Teleport :to="toolbarContainer">
-      <FormattingToolbar :formatting="formatting" @pointerdown.prevent />
-    </Teleport>
-    <LexicalContent ref="contentElement" @wheel.stop.passive @contextmenu.stop @pointerdown.stop />
-    <FloatingSelectionMenu :selectionElement="contentElement">
-      <template #default="{ collapsed }">
-        <SelectionFormattingToolbar v-if="!collapsed" :formatting="formatting" />
-        <LinkToolbar v-else-if="urlUnderCursor" :url="urlUnderCursor" />
-      </template>
-    </FloatingSelectionMenu>
-    <LexicalDecorators :editor="editor" />
-  </div>
+  <EditorRoot
+    ref="editorRoot"
+    class="MarkdownEditor"
+    :class="{ editing }"
+    @focusout="editing = false"
+  />
+  <VueComponentHost ref="vueHost" />
 </template>
 
 <style scoped>
-.fullHeight {
-  display: flex;
-  flex-direction: column;
-  height: 100%;
+:deep(.cm-content) {
+  font-family: var(--font-sans);
 }
 
-:deep(.toggledOn) {
+:deep(.cm-scroller) {
+  /* Prevent touchpad back gesture, which can be triggered while panning. */
+  overscroll-behavior: none;
+}
+
+.EditorRoot :deep(.cm-editor) {
+  position: relative;
+  width: 100%;
+  height: 100%;
+  opacity: 1;
   color: black;
-  opacity: 0.6;
-}
-:deep(.toggledOff) {
-  color: black;
-  opacity: 0.3;
-}
-:deep(.DropdownMenuButton) {
-  color: inherit;
-  opacity: inherit;
-}
-:deep(.DropdownMenuContent .MenuButton) {
-  justify-content: unset;
+  font-size: 12px;
+  outline: none;
 }
 </style>
 
-<style module="lexicalTheme" src="@/components/MarkdownEditor/theme.css" />
+<!--suppress CssUnusedSymbol -->
+<style module>
+/* === Syntax styles === */
+
+.heading1 {
+  font-weight: 700;
+  font-size: 20px;
+  line-height: 1.75;
+}
+.heading2 {
+  font-weight: 700;
+  font-size: 16px;
+  line-height: 1.75;
+}
+.heading3,
+.heading4,
+.heading5,
+.heading6 {
+  font-size: 14px;
+  line-height: 2;
+}
+.processingInstruction {
+  opacity: 20%;
+}
+.emphasis:not(.processingInstruction) {
+  font-style: italic;
+}
+.strong:not(.processingInstruction) {
+  font-weight: bold;
+}
+.strikethrough:not(.processingInstruction) {
+  text-decoration: line-through;
+}
+.monospace {
+  /*noinspection CssNoGenericFontName*/
+  font-family: var(--font-mono);
+}
+
+/* === Editing-mode === */
+
+/* There are currently no style overrides for editing mode, so this is commented out to appease the Vue linter. */
+/* :global(.MarkdownEditor):global(.editing) :global(.cm-line):global(.cm-has-cursor) {} */
+
+/* === View-mode === */
+
+:global(.MarkdownEditor):not(:global(.editing)) :global(.cm-line),
+:global(.cm-line):not(:global(.cm-has-cursor)) {
+  :global(.cm-image-markup) {
+    display: none;
+  }
+  .processingInstruction {
+    display: none;
+  }
+  .url {
+    display: none;
+  }
+  a > .link {
+    display: inline;
+    cursor: pointer;
+    color: #555;
+    &:hover {
+      text-decoration: underline;
+    }
+  }
+  &:has(.list.processingInstruction) {
+    display: list-item;
+    list-style-type: disc;
+    list-style-position: inside;
+  }
+}
+</style>

app/gui/src/project-view/components/MarkdownEditor/SelectionFormattingToolbar.vue

@@ -1,41 +0,0 @@
-<script setup lang="ts">
-import { type UseFormatting } from '@/components/MarkdownEditor/formatting'
-import ToggleIcon from '@/components/ToggleIcon.vue'
-
-const props = defineProps<{ formatting: UseFormatting }>()
-
-const { bold, italic, strikethrough } = props.formatting
-</script>
-
-<template>
-  <div class="SelectionFormattingToolbar">
-    <ToggleIcon
-      :modelValue="bold.state.value"
-      icon="bold"
-      title="Bold"
-      @update:modelValue="bold.set"
-    />
-    <ToggleIcon
-      :modelValue="italic.state.value"
-      icon="italic"
-      title="Italic"
-      @update:modelValue="italic.set"
-    />
-    <ToggleIcon
-      :modelValue="strikethrough.state.value"
-      icon="strike-through"
-      title="Strikethrough"
-      @update:modelValue="strikethrough.set"
-    />
-  </div>
-</template>
-
-<style scoped>
-.SelectionFormattingToolbar {
-  display: flex;
-  background-color: white;
-  border-radius: var(--radius-full);
-  padding: 4px;
-  gap: 4px;
-}
-</style>

app/gui/src/project-view/components/MarkdownEditor/__tests__/markdown.test.ts

@@ -0,0 +1,137 @@
+import { ensoMarkdown } from '@/components/MarkdownEditor/markdown'
+import { EditorState } from '@codemirror/state'
+import { Decoration, EditorView } from '@codemirror/view'
+import { expect, test } from 'vitest'
+
+function decorations<T>(
+  source: string,
+  recognize: (from: number, to: number, decoration: Decoration) => T | undefined,
+) {
+  const vueHost = {
+    register: () => ({ unregister: () => {} }),
+  }
+  const state = EditorState.create({
+    doc: source,
+    extensions: [ensoMarkdown({ vueHost })],
+  })
+  const view = new EditorView({ state })
+  const decorationSets = state.facet(EditorView.decorations)
+  const results = []
+  for (const decorationSet of decorationSets) {
+    const resolvedDecorations =
+      decorationSet instanceof Function ? decorationSet(view) : decorationSet
+    const cursor = resolvedDecorations.iter()
+    while (cursor.value != null) {
+      const recognized = recognize(cursor.from, cursor.to, cursor.value)
+      if (recognized) results.push(recognized)
+      cursor.next()
+    }
+  }
+  return results
+}
+
+function links(source: string) {
+  return decorations(source, (from, to, deco) => {
+    if (deco.spec.tagName === 'a') {
+      return {
+        text: source.substring(from, to),
+        href: deco.spec.attributes.href,
+      }
+    }
+  })
+}
+
+function images(source: string) {
+  return decorations(source, (from, to, deco) => {
+    if ('widget' in deco.spec && 'props' in deco.spec.widget && 'src' in deco.spec.widget.props) {
+      return {
+        from,
+        to,
+        src: deco.spec.widget.props.src,
+        alt: deco.spec.widget.props.alt,
+      }
+    }
+  })
+}
+
+test.each([
+  {
+    markdown: '[Link text](https://www.example.com/index.html)',
+    expectedLinks: [
+      {
+        text: 'Link text',
+        href: 'https://www.example.com/index.html',
+      },
+    ],
+  },
+  {
+    markdown: '[Unclosed url](https://www.example.com/index.html',
+    expectedLinks: [],
+  },
+  {
+    markdown: '[](https://www.example.com/index.html)',
+    expectedLinks: [],
+  },
+  {
+    markdown: '[With empty URL]()',
+    expectedLinks: [],
+  },
+  {
+    markdown: '[With no URL]',
+    expectedLinks: [],
+  },
+  {
+    markdown: '[Unclosed',
+    expectedLinks: [],
+  },
+])('Link decoration: $markdown', ({ markdown, expectedLinks }) => {
+  expect(links(markdown)).toEqual(expectedLinks)
+  expect(images(markdown)).toEqual([])
+})
+
+test.each([
+  {
+    markdown: '![Image](https://www.example.com/image.avif)',
+    image: {
+      src: 'https://www.example.com/image.avif',
+      alt: 'Image',
+    },
+  },
+  {
+    markdown: '![](https://www.example.com/image.avif)',
+    image: {
+      src: 'https://www.example.com/image.avif',
+      alt: '',
+    },
+  },
+  {
+    markdown: '![Image](https://www.example.com/image.avif',
+    image: null,
+  },
+  {
+    markdown: '![Image]()',
+    image: null,
+  },
+  {
+    markdown: '![Image]',
+    image: null,
+  },
+  {
+    markdown: '![Image',
+    image: null,
+  },
+])('Image decoration: $markdown', ({ markdown, image }) => {
+  expect(links(markdown)).toEqual([])
+  expect(images(markdown)).toEqual(
+    image == null ?
+      []
+    : [
+        {
+          from: markdown.length,
+          to: markdown.length,
+          src: image.src,
+          alt: image.alt,
+        },
+      ],
+  )
+})

app/gui/src/project-view/components/MarkdownEditor/formatting.ts

@@ -1,253 +0,0 @@
-import { lexicalTheme } from '@/components/lexical'
-import { useBufferedWritable } from '@/util/reactivity'
-import { $createCodeNode } from '@lexical/code'
-import {
-  $isListNode,
-  INSERT_ORDERED_LIST_COMMAND,
-  INSERT_UNORDERED_LIST_COMMAND,
-  ListNode,
-} from '@lexical/list'
-import {
-  $createHeadingNode,
-  $createQuoteNode,
-  $isHeadingNode,
-  $isQuoteNode,
-  type HeadingTagType,
-} from '@lexical/rich-text'
-import { $setBlocksType } from '@lexical/selection'
-import { $isTableSelection } from '@lexical/table'
-import {
-  $findMatchingParent,
-  $getNearestBlockElementAncestorOrThrow,
-  $getNearestNodeOfType,
-} from '@lexical/utils'
-import type { EditorThemeClasses, LexicalEditor, RangeSelection, TextFormatType } from 'lexical'
-import {
-  $createParagraphNode,
-  $getSelection,
-  $isRangeSelection,
-  $isRootOrShadowRoot,
-  $isTextNode,
-  COMMAND_PRIORITY_LOW,
-  FORMAT_TEXT_COMMAND,
-  SELECTION_CHANGE_COMMAND,
-} from 'lexical'
-import { ref } from 'vue'
-
-/** TODO: Add docs */
-export function useFormatting(editor: LexicalEditor) {
-  const selectionReaders = new Array<(selection: RangeSelection) => void>()
-  function onReadSelection(reader: (selection: RangeSelection) => void) {
-    selectionReaders.push(reader)
-  }
-  function $readState() {
-    const selection = $getSelection()
-    if ($isRangeSelection(selection)) {
-      for (const reader of selectionReaders) {
-        reader(selection)
-      }
-    }
-  }
-  editor.registerUpdateListener(({ editorState }) => {
-    editorState.read($readState)
-  })
-  editor.registerCommand(
-    SELECTION_CHANGE_COMMAND,
-    (_payload, _newEditor) => {
-      $readState()
-      return false
-    },
-    COMMAND_PRIORITY_LOW,
-  )
-  return {
-    bold: useFormatProperty(editor, 'bold', onReadSelection),
-    italic: useFormatProperty(editor, 'italic', onReadSelection),
-    strikethrough: useFormatProperty(editor, 'strikethrough', onReadSelection),
-    subscript: useFormatProperty(editor, 'subscript', onReadSelection),
-    superscript: useFormatProperty(editor, 'superscript', onReadSelection),
-    blockType: useBlockType(editor, onReadSelection),
-    clearFormatting: () => editor.update($clearFormatting),
-  }
-}
-export type UseFormatting = ReturnType<typeof useFormatting>
-
-function useFormatProperty(
-  editor: LexicalEditor,
-  property: TextFormatType,
-  onReadSelection: ($readSelection: (selection: RangeSelection) => void) => void,
-) {
-  const state = ref(false)
-
-  onReadSelection((selection) => (state.value = selection.hasFormat(property)))
-
-  // The editor only exposes a toggle interface, so we need to model its state to ensure the setter is only called when
-  // the desired value is different from its current value.
-  const writable = useBufferedWritable({
-    get: state,
-    set: (_value: boolean) => editor.dispatchCommand(FORMAT_TEXT_COMMAND, property),
-  })
-
-  return { state, set: (value: boolean) => (writable.value = value) }
-}
-
-function $clearFormatting() {
-  const selection = $getSelection()
-  if ($isRangeSelection(selection) || $isTableSelection(selection)) {
-    const anchor = selection.anchor
-    const focus = selection.focus
-    const nodes = selection.getNodes()
-    const extractedNodes = selection.extract()
-
-    if (anchor.key === focus.key && anchor.offset === focus.offset) {
-      return
-    }
-
-    nodes.forEach((node, idx) => {
-      // We split the first and last node by the selection
-      // So that we don't format unselected text inside those nodes
-      if ($isTextNode(node)) {
-        // Use a separate variable to ensure TS does not lose the refinement
-        let textNode = node
-        if (idx === 0 && anchor.offset !== 0) {
-          textNode = textNode.splitText(anchor.offset)[1] || textNode
-        }
-        if (idx === nodes.length - 1) {
-          textNode = textNode.splitText(focus.offset)[0] || textNode
-        }
-        /**
-         * If the selected text has one format applied
-         * selecting a portion of the text, could
-         * clear the format to the wrong portion of the text.
-         *
-         * The cleared text is based on the length of the selected text.
-         */
-        // We need this in case the selected text only has one format
-        const extractedTextNode = extractedNodes[0]
-        if (nodes.length === 1 && $isTextNode(extractedTextNode)) {
-          textNode = extractedTextNode
-        }
-
-        if (textNode.__style !== '') {
-          textNode.setStyle('')
-        }
-        if (textNode.__format !== 0) {
-          textNode.setFormat(0)
-          $getNearestBlockElementAncestorOrThrow(textNode).setFormat('')
-        }
-        node = textNode
-      } else if ($isHeadingNode(node) || $isQuoteNode(node)) {
-        node.replace($createParagraphNode(), true)
-      }
-    })
-  }
-}
-
-export const blockTypeToBlockName = {
-  bullet: 'Bulleted List',
-  code: 'Code Block',
-  h1: 'Heading 1',
-  h2: 'Heading 2',
-  h3: 'Heading 3',
-  number: 'Numbered List',
-  paragraph: 'Normal',
-  quote: 'Quote',
-}
-export type BlockType = keyof typeof blockTypeToBlockName
-export const blockTypes = Object.keys(blockTypeToBlockName) as BlockType[]
-
-const smallestEnabledHeading = ['h6', 'h5', 'h4', 'h3', 'h2', 'h1'].find(
-  isBlockType,
-) as HeadingTagType & BlockType
-
-function isBlockType(value: string): value is BlockType {
-  return value in blockTypeToBlockName
-}
-
-/** TODO: Add docs */
-export function normalizeHeadingLevel(heading: HeadingTagType): HeadingTagType & BlockType {
-  return isBlockType(heading) ? heading : smallestEnabledHeading
-}
-
-function useBlockType(
-  editor: LexicalEditor,
-  onReadSelection: ($readSelection: (selection: RangeSelection) => void) => void,
-) {
-  const state = ref<BlockType>('paragraph')
-
-  onReadSelection((selection) => (state.value = $getBlockType(selection) ?? 'paragraph'))
-
-  function $getBlockType(selection: RangeSelection): BlockType | undefined {
-    const anchorNode = selection.anchor.getNode()
-    const element =
-      anchorNode.getKey() === 'root' ?
-        anchorNode
-      : $findMatchingParent(anchorNode, (e) => {
-          const parent = e.getParent()
-          return parent !== null && $isRootOrShadowRoot(parent)
-        }) ?? anchorNode.getTopLevelElementOrThrow()
-
-    if ($isListNode(element)) {
-      const parentList = $getNearestNodeOfType<ListNode>(anchorNode, ListNode)
-      const type = parentList ? parentList.getListType() : element.getListType()
-      if (type in blockTypeToBlockName) {
-        return type as keyof typeof blockTypeToBlockName
-      }
-    } else if ($isHeadingNode(element)) {
-      return normalizeHeadingLevel(element.getTag())
-    } else {
-      const type = element.getType()
-      if (type in blockTypeToBlockName) {
-        return type as keyof typeof blockTypeToBlockName
-      }
-    }
-  }
-
-  const $setBlockType: Record<BlockType, () => void> = {
-    bullet: () => editor.dispatchCommand(INSERT_UNORDERED_LIST_COMMAND, undefined),
-    number: () => editor.dispatchCommand(INSERT_ORDERED_LIST_COMMAND, undefined),
-    paragraph: () => {
-      const selection = $getSelection()
-      if ($isRangeSelection(selection)) {
-        $setBlocksType(selection, () => $createParagraphNode())
-      }
-    },
-    quote: () => $setBlocksType($getSelection(), () => $createQuoteNode()),
-    code: () => {
-      let selection = $getSelection()
-      if (selection !== null) {
-        if (selection.isCollapsed()) {
-          $setBlocksType(selection, () => $createCodeNode())
-        } else {
-          const textContent = selection.getTextContent()
-          const codeNode = $createCodeNode()
-          selection.insertNodes([codeNode])
-          selection = $getSelection()
-          if ($isRangeSelection(selection)) {
-            selection.insertRawText(textContent)
-          }
-        }
-      }
-    },
-    h1: () => $setBlocksType($getSelection(), () => $createHeadingNode('h1')),
-    h2: () => $setBlocksType($getSelection(), () => $createHeadingNode('h2')),
-    h3: () => $setBlocksType($getSelection(), () => $createHeadingNode('h3')),
-  }
-
-  return {
-    state,
-    set: (value: BlockType) => editor.update($setBlockType[value]),
-  }
-}
-
-/** TODO: Add docs */
-export function lexicalRichTextTheme(themeCss: Record<string, string>): EditorThemeClasses {
-  const theme = lexicalTheme(themeCss)
-  if (theme.heading) {
-    for (const level of Object.keys(theme.heading)) {
-      const levelTag = level as keyof typeof theme.heading
-      const normalized = normalizeHeadingLevel(levelTag)
-      theme.heading[levelTag] = theme.heading[normalized] ?? 'lexical-unsupported-heading-level'
-    }
-  }
-  return theme
-}

app/gui/src/project-view/components/MarkdownEditor/highlight.ts

@@ -0,0 +1,108 @@
+import { syntaxHighlighting } from '@codemirror/language'
+import type { Extension } from '@codemirror/state'
+import { type Tag, tagHighlighter, tags } from '@lezer/highlight'
+
+const tagNames: (keyof typeof tags)[] = [
+  'comment',
+  'lineComment',
+  'blockComment',
+  'docComment',
+  'name',
+  'variableName',
+  'typeName',
+  'tagName',
+  'propertyName',
+  'attributeName',
+  'className',
+  'labelName',
+  'namespace',
+  'macroName',
+  'literal',
+  'string',
+  'docString',
+  'character',
+  'attributeValue',
+  'number',
+  'integer',
+  'float',
+  'bool',
+  'regexp',
+  'escape',
+  'color',
+  'url',
+  'keyword',
+  'self',
+  'null',
+  'atom',
+  'unit',
+  'modifier',
+  'operatorKeyword',
+  'controlKeyword',
+  'definitionKeyword',
+  'moduleKeyword',
+  'operator',
+  'derefOperator',
+  'arithmeticOperator',
+  'logicOperator',
+  'bitwiseOperator',
+  'compareOperator',
+  'updateOperator',
+  'definitionOperator',
+  'typeOperator',
+  'controlOperator',
+  'punctuation',
+  'separator',
+  'bracket',
+  'angleBracket',
+  'squareBracket',
+  'paren',
+  'brace',
+  'content',
+  'heading',
+  'heading1',
+  'heading2',
+  'heading3',
+  'heading4',
+  'heading5',
+  'heading6',
+  'contentSeparator',
+  'list',
+  'quote',
+  'emphasis',
+  'strong',
+  'link',
+  'monospace',
+  'strikethrough',
+  'inserted',
+  'deleted',
+  'changed',
+  'invalid',
+  'meta',
+  'documentMeta',
+  'annotation',
+  'processingInstruction',
+]
+
+/**
+ * Defines an {@link Extension} that applies a highlighting CSS class for any {@link Tag} with a provided class mapping.
+ * @param css A mapping from {@link Tag} names to CSS class names.
+ */
+export function highlightStyle(css: Record<string, string>): Extension {
+  const modTagClasses = (mod: keyof typeof tags) =>
+    tagNames.map((tag) => ({
+      tag: (tags[mod] as any)(tags[tag]) as Tag,
+      class: `${tags[mod]}-${tag}`,
+    }))
+  const tagClasses = tagNames.map((tag) => ({ tag: tags[tag] as Tag, class: css[tag] ?? tag }))
+  return syntaxHighlighting(
+    tagHighlighter([
+      ...tagClasses,
+      ...modTagClasses('definition'),
+      ...modTagClasses('constant'),
+      ...modTagClasses('function'),
+      ...modTagClasses('standard'),
+      ...modTagClasses('local'),
+      ...modTagClasses('special'),
+    ]),
+  )
+}

app/gui/src/project-view/components/MarkdownEditor/imageUrlTransformer.ts

@@ -7,11 +7,11 @@ export type TransformUrlResult = Result<{ url: string; dispose?: () => void }>
 export type UrlTransformer = (url: string) => Promise<TransformUrlResult>
 
 export {
-  injectFn as injectLexicalImageUrlTransformer,
-  provideFn as provideLexicalImageUrlTransformer,
+  injectFn as injectDocumentationImageUrlTransformer,
+  provideFn as provideDocumentationImageUrlTransformer,
 }
 const { provideFn, injectFn } = createContextStore(
-  'Lexical image URL transformer',
+  'Documentation image URL transformer',
   (transformUrl: ToValue<UrlTransformer | undefined>) => ({
     transformUrl: (url: string) => toValue(transformUrl)?.(url),
   }),
@@ -59,7 +59,10 @@ export function fetcherUrlTransformer<ResourceLocation>(
       return Ok({
         url: result.value.value,
         dispose: () => {
-          if (!(result.value.refs -= 1)) URL.revokeObjectURL(result.value.value)
+          if (!(result.value.refs -= 1)) {
+            URL.revokeObjectURL(result.value.value)
+            allocatedUrls.delete(uniqueId)
+          }
         },
       })
     }

app/gui/src/project-view/components/MarkdownEditor/listPlugin.ts

@@ -1,47 +0,0 @@
-import type { LexicalPlugin } from '@/components/lexical'
-import {
-  $handleListInsertParagraph,
-  INSERT_ORDERED_LIST_COMMAND,
-  INSERT_UNORDERED_LIST_COMMAND,
-  insertList,
-  ListItemNode,
-  ListNode,
-  REMOVE_LIST_COMMAND,
-  removeList,
-} from '@lexical/list'
-import { COMMAND_PRIORITY_LOW, INSERT_PARAGRAPH_COMMAND } from 'lexical'
-
-export const listPlugin: LexicalPlugin = {
-  nodes: [ListItemNode, ListNode],
-  register: (editor) => {
-    editor.registerCommand(
-      INSERT_ORDERED_LIST_COMMAND,
-      () => {
-        insertList(editor, 'number')
-        return true
-      },
-      COMMAND_PRIORITY_LOW,
-    )
-    editor.registerCommand(
-      INSERT_UNORDERED_LIST_COMMAND,
-      () => {
-        insertList(editor, 'bullet')
-        return true
-      },
-      COMMAND_PRIORITY_LOW,
-    )
-    editor.registerCommand(
-      REMOVE_LIST_COMMAND,
-      () => {
-        removeList(editor)
-        return true
-      },
-      COMMAND_PRIORITY_LOW,
-    )
-    editor.registerCommand(
-      INSERT_PARAGRAPH_COMMAND,
-      $handleListInsertParagraph,
-      COMMAND_PRIORITY_LOW,
-    )
-  },
-}

app/gui/src/project-view/components/MarkdownEditor/markdown.ts

@@ -1,70 +1,9 @@
-import type { LexicalPlugin } from '@/components/lexical'
-import { useLexicalStringSync } from '@/components/lexical/sync'
-import { CodeHighlightNode, CodeNode } from '@lexical/code'
-import { AutoLinkNode, LinkNode } from '@lexical/link'
-import { ListItemNode, ListNode } from '@lexical/list'
-import {
-  $convertFromMarkdownString,
-  $convertToMarkdownString,
-  TRANSFORMERS,
-  registerMarkdownShortcuts,
-  type Transformer,
-} from '@lexical/markdown'
-import { HeadingNode, QuoteNode, registerRichText } from '@lexical/rich-text'
-import { TableCellNode, TableNode, TableRowNode } from '@lexical/table'
-import { $setSelection } from 'lexical'
-import { watch, type Ref } from 'vue'
+import { markdownDecorators } from '@/components/MarkdownEditor/markdown/decoration'
+import { markdown } from '@/components/MarkdownEditor/markdown/parse'
+import type { VueHost } from '@/components/VueComponentHost.vue'
+import type { Extension } from '@codemirror/state'
 
-export interface LexicalMarkdownPlugin extends LexicalPlugin {
-  transformers?: Transformer[]
+/** Markdown extension, with customizations for Enso. */
+export function ensoMarkdown({ vueHost }: { vueHost: VueHost }): Extension {
+  return [markdown(), markdownDecorators({ vueHost })]
 }
-
-/** TODO: Add docs */
-export function markdownPlugin(
-  model: Ref<string>,
-  extensions: LexicalMarkdownPlugin[],
-): LexicalPlugin[] {
-  const transformers = new Array<Transformer>()
-  for (const extension of extensions) {
-    if (extension?.transformers) transformers.push(...extension.transformers)
-  }
-  transformers.push(...TRANSFORMERS)
-  return [...extensions, baseMarkdownPlugin(transformers), markdownSyncPlugin(model, transformers)]
-}
-
-function baseMarkdownPlugin(transformers: Transformer[]): LexicalPlugin {
-  return {
-    nodes: [
-      HeadingNode,
-      QuoteNode,
-      ListItemNode,
-      ListNode,
-      AutoLinkNode,
-      LinkNode,
-      CodeHighlightNode,
-      CodeNode,
-      TableCellNode,
-      TableNode,
-      TableRowNode,
-    ],
-    register: (editor) => {
-      registerRichText(editor)
-      registerMarkdownShortcuts(editor, transformers)
-    },
-  }
-}
-
-const markdownSyncPlugin = (model: Ref<string>, transformers: Transformer[]): LexicalPlugin => ({
-  register: (editor) => {
-    const { content } = useLexicalStringSync(
-      editor,
-      () => $convertToMarkdownString(transformers),
-      (value) => {
-        $convertFromMarkdownString(value, transformers)
-        $setSelection(null)
-      },
-    )
-    watch(model, (newContent) => content.set(newContent), { immediate: true })
-    watch(content.editedContent, (newContent) => (model.value = newContent))
-  },
-})

app/gui/src/project-view/components/MarkdownEditor/markdown/decoration.ts

@@ -0,0 +1,270 @@
+import DocumentationImage from '@/components/MarkdownEditor/DocumentationImage.vue'
+import type { VueHost } from '@/components/VueComponentHost.vue'
+import { syntaxTree } from '@codemirror/language'
+import { type EditorSelection, type Extension, RangeSetBuilder, type Text } from '@codemirror/state'
+import {
+  Decoration,
+  type DecorationSet,
+  EditorView,
+  type PluginValue,
+  ViewPlugin,
+  type ViewUpdate,
+  WidgetType,
+} from '@codemirror/view'
+import type { SyntaxNode, SyntaxNodeRef, Tree } from '@lezer/common'
+import { h, markRaw } from 'vue'
+
+/** Extension applying decorators for Markdown. */
+export function markdownDecorators({ vueHost }: { vueHost: VueHost }): Extension {
+  const stateDecorator = new TreeStateDecorator(vueHost, [
+    decorateImageWithClass,
+    decorateImageWithRendered,
+  ])
+  const stateDecoratorExt = EditorView.decorations.compute(['doc'], (state) =>
+    stateDecorator.decorate(syntaxTree(state), state.doc),
+  )
+  const viewDecoratorExt = ViewPlugin.define(
+    (view) => new TreeViewDecorator(view, vueHost, [decorateLink]),
+    {
+      decorations: (v) => v.decorations,
+    },
+  )
+  const cursorDecoratorExt = EditorView.decorations.compute(['selection', 'doc'], (state) =>
+    cursorDecorations(state.selection, state.doc),
+  )
+  return [stateDecoratorExt, viewDecoratorExt, cursorDecoratorExt]
+}
+
+interface NodeDecorator {
+  (
+    nodeRef: SyntaxNodeRef,
+    doc: Text,
+    emitDecoration: (from: number, to: number, deco: Decoration) => void,
+    vueHost: VueHost,
+  ): void
+}
+
+// === Tree state decorator ===
+
+/** Maintains a set of decorations based on the tree. */
+class TreeStateDecorator {
+  constructor(
+    private readonly vueHost: VueHost,
+    private readonly nodeDecorators: NodeDecorator[],
+  ) {}
+
+  decorate(tree: Tree, doc: Text): DecorationSet {
+    const builder = new RangeSetBuilder<Decoration>()
+    const emit = (from: number, to: number, value: Decoration) => {
+      builder.add(from, to, value)
+    }
+    tree.iterate({
+      enter: (nodeRef) => {
+        for (const decorator of this.nodeDecorators) decorator(nodeRef, doc, emit, this.vueHost)
+      },
+    })
+    return builder.finish()
+  }
+}
+
+// === Cursor decorator ===
+
+function cursorDecorations(selection: EditorSelection, doc: Text): DecorationSet {
+  const builder = new RangeSetBuilder<Decoration>()
+  for (const range of selection.ranges) {
+    const line = doc.lineAt(range.from)
+    builder.add(
+      line.from,
+      line.from,
+      Decoration.line({
+        class: 'cm-has-cursor',
+      }),
+    )
+    if (range.to != range.from) {
+      // TODO: Add decorations to each line
+    }
+  }
+  return builder.finish()
+}
+
+// === Tree view decorator ===
+
+/** Maintains a set of decorations based on the tree, lazily-constructed for the visible range of the document. */
+class TreeViewDecorator implements PluginValue {
+  decorations: DecorationSet
+
+  constructor(
+    view: EditorView,
+    private readonly vueHost: VueHost,
+    /**
+     * Functions that construct decorations based on tree. The decorations must not have significant impact on the
+     * height of the document, or scrolling issues would result, because decorations are lazily computed based on the
+     * current viewport.
+     */
+    private readonly nodeDecorators: NodeDecorator[],
+  ) {
+    this.decorations = this.buildDeco(syntaxTree(view.state), view)
+  }
+
+  update(update: ViewUpdate) {
+    // TODO
+    // Attaching widgets can change the geometry, so don't re-attach widgets in response to geometry changes.
+    // Reusing unchanged widgets would be a better solution, but this works correctly as long as rendering widgets
+    // within the `visibleRanges` doesn't bring any new content into the `visibleRanges`; in practice this should hold.
+    //if (!update.docChanged && !update.viewportChanged) return
+    if (!update.docChanged) return
+    this.decorations = this.buildDeco(syntaxTree(update.state), update.view)
+  }
+
+  private buildDeco(tree: Tree, view: EditorView) {
+    if (!tree.length) return Decoration.none
+    const builder = new RangeSetBuilder<Decoration>()
+    const doc = view.state.doc
+    const emit = (from: number, to: number, value: Decoration) => {
+      builder.add(from, to, value)
+    }
+    for (const { from, to } of view.visibleRanges) {
+      tree.iterate({
+        from,
+        to,
+        enter: (nodeRef) => {
+          for (const decorator of this.nodeDecorators) decorator(nodeRef, doc, emit, this.vueHost)
+        },
+      })
+    }
+    return builder.finish()
+  }
+}
+
+// === Links ===
+
+/** Parse a link or image */
+function parseLinkLike(node: SyntaxNode, doc: Text) {
+  const textOpen = node.firstChild // [ or ![
+  if (!textOpen) return
+  const textClose = textOpen.nextSibling // ]
+  if (!textClose) return
+  const urlOpen = textClose.nextSibling // (
+  // The parser accepts partial links such as `[Missing url]`.
+  if (!urlOpen) return
+  const urlNode = urlOpen.nextSibling
+  // If the URL is empty, this will be the closing 'LinkMark'.
+  if (urlNode?.name !== 'URL') return
+  return {
+    textFrom: textOpen.to,
+    textTo: textClose.from,
+    url: doc.sliceString(urlNode.from, urlNode.to),
+  }
+}
+
+function decorateLink(
+  nodeRef: SyntaxNodeRef,
+  doc: Text,
+  emitDecoration: (from: number, to: number, deco: Decoration) => void,
+) {
+  if (nodeRef.name === 'Link') {
+    const parsed = parseLinkLike(nodeRef.node, doc)
+    if (!parsed) return
+    const { textFrom, textTo, url } = parsed
+    if (textFrom === textTo) return
+    emitDecoration(
+      textFrom,
+      textTo,
+      Decoration.mark({
+        tagName: 'a',
+        attributes: { href: url },
+      }),
+    )
+  }
+}
+
+// === Images ===
+
+function decorateImageWithClass(
+  nodeRef: SyntaxNodeRef,
+  _doc: Text,
+  emitDecoration: (from: number, to: number, deco: Decoration) => void,
+) {
+  if (nodeRef.name === 'Image') {
+    emitDecoration(
+      nodeRef.from,
+      nodeRef.to,
+      Decoration.mark({
+        class: 'cm-image-markup',
+      }),
+    )
+  }
+}
+
+function decorateImageWithRendered(
+  nodeRef: SyntaxNodeRef,
+  doc: Text,
+  emitDecoration: (from: number, to: number, deco: Decoration) => void,
+  vueHost: VueHost,
+) {
+  if (nodeRef.name === 'Image') {
+    const parsed = parseLinkLike(nodeRef.node, doc)
+    if (!parsed) return
+    const { textFrom, textTo, url } = parsed
+    const text = doc.sliceString(textFrom, textTo)
+    const widget = new ImageWidget({ alt: text, src: url }, vueHost)
+    emitDecoration(
+      nodeRef.to,
+      nodeRef.to,
+      Decoration.widget({
+        widget,
+        // Ensure the cursor is drawn relative to the content before the widget.
+        // If it is drawn relative to the widget, it will be hidden when the widget is hidden (i.e. during editing).
+        side: 1,
+      }),
+    )
+  }
+}
+
+class ImageWidget extends WidgetType {
+  private container: HTMLElement | undefined
+  private vueHostRegistration: { unregister: () => void } | undefined
+
+  constructor(
+    private readonly props: {
+      readonly alt: string
+      readonly src: string
+    },
+    private readonly vueHost: VueHost,
+  ) {
+    super()
+  }
+
+  override get estimatedHeight() {
+    return -1
+  }
+
+  override eq(other: WidgetType) {
+    return (
+      other instanceof ImageWidget &&
+      other.props.src == this.props.src &&
+      other.props.alt == this.props.alt
+    )
+  }
+
+  override toDOM(): HTMLElement {
+    if (!this.container) {
+      const container = markRaw(document.createElement('span'))
+      container.className = 'cm-image-rendered'
+      this.vueHostRegistration = this.vueHost.register(
+        h(DocumentationImage, {
+          src: this.props.src,
+          alt: this.props.alt,
+        }),
+        container,
+      )
+      this.container = container
+    }
+    return this.container
+  }
+
+  override destroy() {
+    this.vueHostRegistration?.unregister()
+    this.container = undefined
+  }
+}

app/gui/src/project-view/components/MarkdownEditor/markdown/lezer.d.ts

@@ -0,0 +1,35 @@
+/** @file Private lezer-markdown symbols used by lezer-markdown parsers we have customized versions of. */
+
+import { Tree, TreeBuffer } from '@lezer/common'
+import { Element } from '@lezer/markdown'
+
+declare module '@lezer/markdown' {
+  export interface BlockContext {
+    block: CompositeBlock
+    stack: CompositeBlock[]
+    readonly buffer: Buffer
+
+    addNode: (block: number | Tree, from: number, to?: number) => void
+    startContext: (type: number, start: number, value?: number) => void
+  }
+
+  export interface CompositeBlock {
+    readonly type: number
+    // Used for indentation in list items, markup character in lists
+    readonly value: number
+    readonly from: number
+    readonly hash: number
+    end: number
+    readonly children: (Tree | TreeBuffer)[]
+    readonly positions: number[]
+  }
+
+  export interface Buffer {
+    content: number[]
+    nodes: Tree[]
+
+    write: (type: number, from: number, to: number, children?: number) => Buffer
+    writeElements: (elts: readonly Element[], offset?: number) => Buffer
+    finish: (type: number, length: number) => Tree
+  }
+}

app/gui/src/project-view/components/MarkdownEditor/markdown/parse.ts

@@ -0,0 +1,248 @@
+import { markdown as baseMarkdown, markdownLanguage } from '@codemirror/lang-markdown'
+import type { Extension } from '@codemirror/state'
+import type { Tree } from '@lezer/common'
+import type { BlockContext, BlockParser, Line, MarkdownParser, NodeSpec } from '@lezer/markdown'
+import { Element } from '@lezer/markdown'
+import { assertDefined } from 'ydoc-shared/util/assert'
+
+/**
+ * Enso Markdown extension. Differences from CodeMirror's base Markdown extension:
+ * - It defines the flavor of Markdown supported in Enso documentation. Currently, this is mostly CommonMark except we
+ *   don't support setext headings. Planned features include support for some GFM extensions.
+ * - Many of the parsers differ from the `@lezer/markdown` parsers in their treatment of whitespace, in order to support
+ *   a rendering mode where markup (and some associated spacing) is hidden.
+ */
+export function markdown(): Extension {
+  return baseMarkdown({
+    base: markdownLanguage,
+    extensions: [
+      {
+        parseBlock: [headerParser, bulletList, orderedList, blockquoteParser, disableSetextHeading],
+        defineNodes: [blockquoteNode],
+      },
+    ],
+  })
+}
+
+function getType({ parser }: { parser: MarkdownParser }, name: string) {
+  const ty = parser.nodeSet.types.find((ty) => ty.name === name)
+  assertDefined(ty)
+  return ty.id
+}
+
+/** Parser override to include the space in the delimiter. */
+const headerParser: BlockParser = {
+  name: 'ATXHeading',
+  parse: (cx, line) => {
+    let size = isAtxHeading(line)
+    if (size < 0) return false
+    const level = size
+    // If the character after the hashes is a space, treat it as part of the `HeaderMark`.
+    if (isSpace(line.text.charCodeAt(size))) size += 1
+    const off = line.pos
+    const from = cx.lineStart + off
+    // Trailing spaces at EOL
+    const endOfSpace = skipSpaceBack(line.text, line.text.length, off)
+    let after = endOfSpace
+    // Trailing sequence of # (before EOL spaces)
+    while (after > off && line.text.charCodeAt(after - 1) == line.next) after--
+    if (after == endOfSpace || after == off || !isSpace(line.text.charCodeAt(after - 1)))
+      after = line.text.length
+    const headerMark = getType(cx, 'HeaderMark')
+    const buf = cx.buffer
+      .write(headerMark, 0, size)
+      .writeElements(cx.parser.parseInline(line.text.slice(off + size, after), from + size), -from)
+    if (after < line.text.length) buf.write(headerMark, after - off, endOfSpace - off)
+    const node = buf.finish(getType(cx, `ATXHeading${level}`), line.text.length - off)
+    cx.nextLine()
+    cx.addNode(node, from)
+    return true
+  },
+}
+
+/** Parser override to include the space in the delimiter. */
+const bulletList: BlockParser = {
+  name: 'BulletList',
+  parse: (cx, line) => {
+    const size = isBulletList(line, cx, false)
+    if (size < 0) return false
+    const length = size + (isSpace(line.text.charCodeAt(line.pos + 1)) ? 1 : 0)
+    const bulletList = getType(cx, 'BulletList')
+    if (cx.block.type != bulletList) cx.startContext(bulletList, line.basePos, line.next)
+    const newBase = getListIndent(line, line.pos + 1)
+    cx.startContext(getType(cx, 'ListItem'), line.basePos, newBase - line.baseIndent)
+    cx.addNode(getType(cx, 'ListMark'), cx.lineStart + line.pos, cx.lineStart + line.pos + length)
+    line.moveBaseColumn(newBase)
+    return null
+  },
+}
+
+/** Parser override to include the space in the delimiter. */
+const orderedList: BlockParser = {
+  name: 'OrderedList',
+  parse: (cx, line) => {
+    const size = isOrderedList(line, cx, false)
+    if (size < 0) return false
+    const orderedList = getType(cx, 'OrderedList')
+    if (cx.block.type != orderedList)
+      cx.startContext(orderedList, line.basePos, line.text.charCodeAt(line.pos + size - 1))
+    const newBase = getListIndent(line, line.pos + size)
+    cx.startContext(getType(cx, 'ListItem'), line.basePos, newBase - line.baseIndent)
+    cx.addNode(getType(cx, 'ListMark'), cx.lineStart + line.pos, cx.lineStart + line.pos + size)
+    line.moveBaseColumn(newBase)
+    return null
+  },
+}
+
+const ENSO_BLOCKQUOTE_TYPE = 'EnsoBlockquote'
+
+/** Parser override to include the space in the delimiter. */
+const blockquoteParser: BlockParser = {
+  name: ENSO_BLOCKQUOTE_TYPE,
+  parse: (cx, line) => {
+    const size = isBlockquote(line)
+    if (size < 0) return false
+    const type = getType(cx, ENSO_BLOCKQUOTE_TYPE)
+    cx.startContext(type, line.pos)
+    cx.addNode(getType(cx, 'QuoteMark'), cx.lineStart + line.pos, cx.lineStart + line.pos + size)
+    line.moveBase(line.pos + size)
+    return null
+  },
+  before: 'Blockquote',
+}
+
+/**
+ * Replaces setext heading parser with a parser that never matches.
+ *
+ * When starting a bulleted list, the `SetextHeading` parser can match when a `-` has been typed and a following space
+ * hasn't been entered yet; the resulting style changes are distracting. To prevent this, we don't support setext
+ * headings; ATX headings seem to be much more popular anyway.
+ */
+const disableSetextHeading: BlockParser = {
+  name: 'SetextHeading',
+  parse: () => false,
+}
+
+const blockquoteNode: NodeSpec = {
+  name: ENSO_BLOCKQUOTE_TYPE,
+  block: true,
+  composite: (cx, line) => {
+    if (line.next != 62 /* '>' */) return false
+    const size = isSpace(line.text.charCodeAt(line.pos + 1)) ? 2 : 1
+    line.addMarker(
+      elt(getType(cx, 'QuoteMark'), cx.lineStart + line.pos, cx.lineStart + line.pos + size),
+    )
+    line.moveBase(line.pos + size)
+    //bl.end = cx.lineStart + line.text.length
+    return true
+  },
+}
+
+function elt(type: number, from: number, to: number): Element {
+  return new (Element as any)(type, from, to)
+}
+
+function isBlockquote(line: Line) {
+  return (
+    line.next != 62 /* '>' */ ? -1
+    : line.text.charCodeAt(line.pos + 1) == 32 ? 2
+    : 1
+  )
+}
+
+function isBulletList(line: Line, cx: BlockContext, breaking: boolean) {
+  return (
+      (line.next == 45 || line.next == 43 || line.next == 42) /* '-+*' */ &&
+        (line.pos == line.text.length - 1 || isSpace(line.text.charCodeAt(line.pos + 1))) &&
+        (!breaking || inList(cx, 'BulletList') || line.skipSpace(line.pos + 2) < line.text.length)
+    ) ?
+      1
+    : -1
+}
+
+function isOrderedList(line: Line, cx: BlockContext, breaking: boolean) {
+  let pos = line.pos
+  let next = line.next
+  for (;;) {
+    if (next >= 48 && next <= 57 /* '0-9' */) pos++
+    else break
+    if (pos == line.text.length) return -1
+    next = line.text.charCodeAt(pos)
+  }
+  if (
+    pos == line.pos ||
+    pos > line.pos + 9 ||
+    (next != 46 && next != 41) /* '.)' */ ||
+    (pos < line.text.length - 1 && !isSpace(line.text.charCodeAt(pos + 1))) ||
+    (breaking &&
+      !inList(cx, 'OrderedList') &&
+      (line.skipSpace(pos + 1) == line.text.length ||
+        pos > line.pos + 1 ||
+        line.next != 49)) /* '1' */
+  )
+    return -1
+  return pos + 1 - line.pos
+}
+
+function inList(cx: BlockContext, typeName: string) {
+  const type = getType(cx, typeName)
+  for (let i = cx.stack.length - 1; i >= 0; i--) if (cx.stack[i]!.type == type) return true
+  return false
+}
+
+function getListIndent(line: Line, pos: number) {
+  const indentAfter = line.countIndent(pos, line.pos, line.indent)
+  const indented = line.countIndent(line.skipSpace(pos), pos, indentAfter)
+  return indented >= indentAfter + 5 ? indentAfter + 1 : indented
+}
+
+// === Debugging ===
+
+/** Represents the structure of a @{link Tree} in a JSON-compatible format. */
+export interface DebugTree {
+  /** The name of the {@link NodeType} */
+  name: string
+  children: DebugTree[]
+}
+
+// noinspection JSUnusedGlobalSymbols
+/** @returns A debug representation of the provided {@link Tree} */
+export function debugTree(tree: Tree): DebugTree {
+  const cursor = tree.cursor()
+  let current: DebugTree[] = []
+  const stack: DebugTree[][] = []
+  cursor.iterate(
+    (node) => {
+      const children: DebugTree[] = []
+      current.push({
+        name: node.name,
+        children,
+      })
+      stack.push(current)
+      current = children
+    },
+    () => (current = stack.pop()!),
+  )
+  return current[0]!
+}
+
+// === Helpers ===
+
+function skipSpaceBack(line: string, i: number, to: number) {
+  while (i > to && isSpace(line.charCodeAt(i - 1))) i--
+  return i
+}
+
+/** Returns the number of hash marks at the beginning of the line, or -1 if it is not in the range [1, 6] */
+function isAtxHeading(line: Line) {
+  if (line.next != 35 /* '#' */) return -1
+  let pos = line.pos + 1
+  while (pos < line.text.length && line.text.charCodeAt(pos) == 35) pos++
+  if (pos < line.text.length && line.text.charCodeAt(pos) != 32) return -1
+  const size = pos - line.pos
+  return size > 6 ? -1 : size
+}
+
+function isSpace(ch: number) {
+  return ch == 32 || ch == 9 || ch == 10 || ch == 13
+}

app/gui/src/project-view/components/MarkdownEditor/theme.css

@@ -1,65 +0,0 @@
-/*
-Lexical theme. Class names are derived from the `LexicalThemeClasses` type from `lexical`, with the hierarchy flattened
-using `_` to separate levels. See the `lexicalTheme` function in `lexical/formatting.ts`.
-*/
-
-.heading_h1 {
-  font-weight: 700;
-  font-size: 20px;
-  line-height: 1.75;
-}
-.heading_h2 {
-  font-weight: 700;
-  font-size: 16px;
-  line-height: 1.75;
-}
-.heading_h3,
-.heading_h4,
-.heading_h5,
-.heading_h6 {
-  font-size: 14px;
-  line-height: 2;
-}
-
-.text_strikethrough {
-  text-decoration: line-through;
-}
-.text_italic {
-  font-style: italic;
-}
-.text_bold {
-  font-weight: bold;
-}
-
-.quote {
-  margin-left: 0.2em;
-  border-left: 0.3em solid #ccc;
-  padding-left: 1.6em;
-}
-
-.paragraph {
-  margin-bottom: 0.5em;
-}
-
-.list_ol {
-  list-style-type: decimal;
-  list-style-position: outside;
-  padding-left: 1.6em;
-}
-.list_ul {
-  list-style-type: disc;
-  list-style-position: outside;
-  padding-left: 1.6em;
-}
-
-.image > img {
-  display: inline;
-  margin: 0 0.1em;
-}
-
-.link {
-  color: #555;
-  &:hover {
-    text-decoration: underline;
-  }
-}

app/gui/src/project-view/components/SelectionDropdown.vue

@@ -4,7 +4,7 @@
 import DropdownMenu from '@/components/DropdownMenu.vue'
 import MenuButton from '@/components/MenuButton.vue'
 import SvgIcon from '@/components/SvgIcon.vue'
-import { SelectionMenuOption } from '@/components/visualizations/toolbar'
+import type { SelectionMenuOption } from '@/components/visualizations/toolbar'
 import { ref } from 'vue'
 
 type Key = number | string | symbol

app/gui/src/project-view/components/ToggleIcon.vue

@@ -8,7 +8,7 @@
 
 import MenuButton from '@/components/MenuButton.vue'
 import SvgIcon from '@/components/SvgIcon.vue'
-import { URLString } from '@/util/data/urlString'
+import type { URLString } from '@/util/data/urlString'
 import type { Icon } from '@/util/iconName'
 
 const toggledOn = defineModel<boolean>({ default: false })

app/gui/src/project-view/components/VueComponentHost.vue

@@ -0,0 +1,39 @@
+<script setup lang="ts">
+import { useObjectId } from 'enso-common/src/utilities/data/object'
+import { type Component, reactive } from 'vue'
+
+const teleportations = reactive(new Map<Component, HTMLElement>())
+
+defineExpose({
+  register: (component: Component, element: HTMLElement) => {
+    teleportations.set(component, element)
+    return { unregister: () => teleportations.delete(component) }
+  },
+} satisfies VueHost)
+
+const { objectId } = useObjectId()
+</script>
+
+<script lang="ts">
+/**
+ * Supports creation of Vue Components within a particular Vue context.
+ *
+ * This enables creating Vue Components from code run outside any Vue context by APIs that render custom HTML content
+ * but aren't Vue-aware.
+ */
+export interface VueHost {
+  /**
+   * Request the given component to begin being rendered as a child of the specified HTML element. The returned
+   * `unregister` function should be called when the component should no longer be rendered.
+   */
+  register: (component: Component, element: HTMLElement) => { unregister: () => void }
+}
+</script>
+
+<template>
+  <template v-for="[component, slot] in teleportations.entries()" :key="objectId(component)">
+    <Teleport :to="slot">
+      <component :is="component" />
+    </Teleport>
+  </template>
+</template>

app/gui/src/project-view/components/lexical/LinkPlugin/index.ts

@@ -1,15 +1,9 @@
 import { documentationEditorBindings } from '@/bindings'
-import { IMAGE } from '@/components/MarkdownEditor/ImagePlugin'
-import { $isImageNode } from '@/components/MarkdownEditor/ImagePlugin/imageNode'
-import type { LexicalMarkdownPlugin } from '@/components/MarkdownEditor/markdown'
 import type { LexicalPlugin } from '@/components/lexical'
-import { $createLinkNode, $isLinkNode, AutoLinkNode, LinkNode } from '@lexical/link'
-import type { Transformer } from '@lexical/markdown'
+import { AutoLinkNode, LinkNode } from '@lexical/link'
 import { $getNearestNodeOfType } from '@lexical/utils'
 import {
-  $createTextNode,
   $getSelection,
-  $isTextNode,
   CLICK_COMMAND,
   COMMAND_PRIORITY_CRITICAL,
   COMMAND_PRIORITY_LOW,
@@ -27,51 +21,6 @@ const EMAIL_REGEX =
 
 export const __TEST = { URL_REGEX, EMAIL_REGEX }
 
-const LINK: Transformer = {
-  dependencies: [LinkNode],
-  export: (node, exportChildren, exportFormat) => {
-    if (!$isLinkNode(node)) {
-      return null
-    }
-    const title = node.getTitle()
-    const linkContent =
-      title ?
-        `[${node.getTextContent()}](${node.getURL()} "${title}")`
-      : `[${node.getTextContent()}](${node.getURL()})`
-    const firstChild = node.getFirstChild()
-    // Add text styles only if link has single text node inside. If it's more
-    // then one we ignore it as markdown does not support nested styles for links
-    if (node.getChildrenSize() === 1 && $isTextNode(firstChild)) {
-      return exportFormat(firstChild, linkContent)
-    } else if (node.getChildrenSize() === 1 && $isImageNode(firstChild)) {
-      // Images sometimes happen to be inside links (when importing nodes from HTML).
-      // The link is not important for us (this type of layout is not supported in markdown),
-      // but we want to display the image.
-      return IMAGE.export(firstChild, exportChildren, exportFormat)
-    } else {
-      return linkContent
-    }
-  },
-  importRegExp: /(?:\[([^[]+)\])(?:\((?:([^()\s]+)(?:\s"((?:[^"]*\\")*[^"]*)"\s*)?)\))/,
-  regExp: /(?:\[([^[]+)\])(?:\((?:([^()\s]+)(?:\s"((?:[^"]*\\")*[^"]*)"\s*)?)\))$/,
-  replace: (textNode, match) => {
-    const [, linkText, linkUrl, linkTitle] = match
-    if (linkText && linkUrl) {
-      const linkNode = $createLinkNode(linkUrl, {
-        title: linkTitle ?? null,
-        rel: 'nofollow',
-        target: '_blank',
-      })
-      const linkTextNode = $createTextNode(linkText)
-      linkTextNode.setFormat(textNode.getFormat())
-      linkNode.append(linkTextNode)
-      textNode.replace(linkNode)
-    }
-  },
-  trigger: ')',
-  type: 'text-match',
-}
-
 /** TODO: Add docs */
 export function $getSelectedLinkNode() {
   const selection = $getSelection()
@@ -87,17 +36,6 @@ export function $getSelectedLinkNode() {
   }
 }
 
-const linkClickHandler = documentationEditorBindings.handler({
-  openLink() {
-    const link = $getSelectedLinkNode()
-    if (link instanceof LinkNode) {
-      window.open(link.getURL(), '_blank')?.focus()
-      return true
-    }
-    return false
-  },
-})
-
 const autoLinkClickHandler = documentationEditorBindings.handler({
   openLink() {
     const link = $getSelectedLinkNode()
@@ -109,18 +47,6 @@ const autoLinkClickHandler = documentationEditorBindings.handler({
   },
 })
 
-export const linkPlugin: LexicalMarkdownPlugin = {
-  nodes: [LinkNode],
-  transformers: [LINK],
-  register(editor: LexicalEditor): void {
-    editor.registerCommand(
-      CLICK_COMMAND,
-      (event) => linkClickHandler(event),
-      COMMAND_PRIORITY_CRITICAL,
-    )
-  },
-}
-
 export const autoLinkPlugin: LexicalPlugin = {
   nodes: [AutoLinkNode],
   register(editor: LexicalEditor): void {

app/gui/src/project-view/components/shared/AgGridTableView.vue

@@ -47,7 +47,7 @@ import {
   tsvTableToEnsoExpression,
   writeClipboard,
 } from '@/components/GraphEditor/clipboard'
-import { TextFormatOptions } from '@/components/visualizations/TableVisualization.vue'
+import type { TextFormatOptions } from '@/components/visualizations/TableVisualization.vue'
 import { useAutoBlur } from '@/util/autoBlur'
 import type {
   CellEditingStartedEvent,
@@ -66,6 +66,8 @@ import type {
   RowHeightParams,
   SortChangedEvent,
 } from 'ag-grid-enterprise'
+import * as iter from 'enso-common/src/utilities/data/iter'
+import { LINE_BOUNDARIES } from 'enso-common/src/utilities/data/string'
 import { type ComponentInstance, reactive, ref, shallowRef, watch } from 'vue'
 
 const DEFAULT_ROW_HEIGHT = 22
@@ -114,14 +116,11 @@ function getRowHeight(params: RowHeightParams): number {
     return DEFAULT_ROW_HEIGHT
   }
 
-  const returnCharsCount = textValues.map((text: string) => {
-    const crlfCount = (text.match(/\r\n/g) || []).length
-    const crCount = (text.match(/\r/g) || []).length
-    const lfCount = (text.match(/\n/g) || []).length
-    return crCount + lfCount - crlfCount
-  })
+  const returnCharsCount = iter.map(textValues, (text) =>
+    iter.count(text.matchAll(LINE_BOUNDARIES)),
+  )
 
-  const maxReturnCharsCount = Math.max(...returnCharsCount)
+  const maxReturnCharsCount = iter.reduce(returnCharsCount, Math.max, 0)
   return (maxReturnCharsCount + 1) * DEFAULT_ROW_HEIGHT
 }
 

app/gui/src/project-view/components/visualizations/TableVisualization.vue

@@ -1,7 +1,7 @@
 <script lang="ts">
 import icons from '@/assets/icons.svg'
 import AgGridTableView, { commonContextMenuActions } from '@/components/shared/AgGridTableView.vue'
-import { SortModel, useTableVizToolbar } from '@/components/visualizations/tableVizToolbar'
+import { useTableVizToolbar, type SortModel } from '@/components/visualizations/tableVizToolbar'
 import { Ast } from '@/util/ast'
 import { Pattern } from '@/util/ast/match'
 import { useVisualizationConfig } from '@/util/visualizationBuiltins'

app/gui/src/project-view/components/visualizations/VisualizationHost.vue

@@ -1,10 +1,10 @@
 <script setup lang="ts">
-import { NodeCreationOptions } from '@/components/GraphEditor/nodeCreation'
+import type { NodeCreationOptions } from '@/components/GraphEditor/nodeCreation'
 import LoadingVisualization from '@/components/visualizations/LoadingVisualization.vue'
-import { ToolbarItem } from '@/components/visualizations/toolbar'
+import type { ToolbarItem } from '@/components/visualizations/toolbar'
 import { provideVisualizationConfig } from '@/providers/visualizationConfig'
-import { Vec2 } from '@/util/data/vec2'
-import { ToValue } from '@/util/reactivity'
+import type { Vec2 } from '@/util/data/vec2'
+import type { ToValue } from '@/util/reactivity'
 
 const { visualization, data, size, nodeType } = defineProps<{
   visualization?: string | object

app/gui/src/project-view/components/visualizations/tableVizToolbar.ts

@@ -1,10 +1,10 @@
-import { NodeCreationOptions } from '@/components/GraphEditor/nodeCreation'
-import { TextFormatOptions } from '@/components/visualizations/TableVisualization.vue'
-import { ToolbarItem } from '@/components/visualizations/toolbar'
+import type { NodeCreationOptions } from '@/components/GraphEditor/nodeCreation'
+import type { TextFormatOptions } from '@/components/visualizations/TableVisualization.vue'
+import type { ToolbarItem } from '@/components/visualizations/toolbar'
 import { Ast } from '@/util/ast'
 import { Pattern } from '@/util/ast/match'
-import { ToValue } from '@/util/reactivity'
-import { computed, ComputedRef, Ref, toValue } from 'vue'
+import type { ToValue } from '@/util/reactivity'
+import { computed, type ComputedRef, type Ref, toValue } from 'vue'
 
 type SortDirection = 'asc' | 'desc'
 export type SortModel = {

app/gui/src/project-view/components/visualizations/toolbar.ts

@@ -1,7 +1,7 @@
-import { URLString } from '@/util/data/urlString'
-import { Icon } from '@/util/iconName'
-import { ToValue } from '@/util/reactivity'
-import { Ref } from 'vue'
+import type { URLString } from '@/util/data/urlString'
+import type { Icon } from '@/util/iconName'
+import type { ToValue } from '@/util/reactivity'
+import type { Ref } from 'vue'
 
 export interface Button {
   icon: Icon | URLString

app/gui/src/project-view/components/widgets/FileBrowserWidget.vue

@@ -3,15 +3,14 @@ import LoadingSpinner from '@/components/shared/LoadingSpinner.vue'
 import SvgButton from '@/components/SvgButton.vue'
 import SvgIcon from '@/components/SvgIcon.vue'
 import { useBackend } from '@/composables/backend'
-import { ToValue } from '@/util/reactivity'
-import Backend, {
-  assetIsDirectory,
-  assetIsFile,
+import type { ToValue } from '@/util/reactivity'
+import type {
   DirectoryAsset,
   DirectoryId,
   FileAsset,
   FileId,
 } from 'enso-common/src/services/Backend'
+import Backend, { assetIsDirectory, assetIsFile } from 'enso-common/src/services/Backend'
 import { computed, ref, toValue, watch } from 'vue'
 
 const emit = defineEmits<{

app/gui/src/project-view/composables/__tests__/nodeCreation.test.ts

@@ -26,7 +26,7 @@ test.each([
 // This is a special case because when a block is empty, adding a line requires adding *two* linebreaks.
 test('Adding node to empty block', () => {
   const module = Ast.MutableModule.Transient()
-  const func = Ast.Function.new(identifier('f')!, [], Ast.BodyBlock.new([], module), {
+  const func = Ast.FunctionDef.new(identifier('f')!, [], Ast.BodyBlock.new([], module), {
     edit: module,
   })
   const rootBlock = Ast.BodyBlock.new([], module)

app/gui/src/project-view/composables/backend.ts

@@ -1,6 +1,7 @@
 import { injectBackend } from '@/providers/backend'
 import type { ToValue } from '@/util/reactivity'
-import { useQuery, useQueryClient, UseQueryOptions, UseQueryReturnType } from '@tanstack/vue-query'
+import type { UseQueryOptions, UseQueryReturnType } from '@tanstack/vue-query'
+import { useQuery, useQueryClient } from '@tanstack/vue-query'
 import type { BackendMethods } from 'enso-common/src/backendQuery'
 import { backendBaseOptions, backendQueryKey } from 'enso-common/src/backendQuery'
 import Backend from 'enso-common/src/services/Backend'

app/gui/src/project-view/composables/navigator.ts

@@ -11,7 +11,7 @@ import type { KeyboardComposable } from '@/composables/keyboard'
 import { Rect } from '@/util/data/rect'
 import { Vec2 } from '@/util/data/vec2'
 import { useEventListener } from '@vueuse/core'
-import { Handler, useGesture } from '@vueuse/gesture'
+import { useGesture, type Handler } from '@vueuse/gesture'
 import {
   computed,
   onScopeDispose,

app/gui/src/project-view/composables/nodeCreation.ts

@@ -12,11 +12,11 @@ import type { Typename } from '@/stores/suggestionDatabase/entry'
 import { Ast } from '@/util/ast'
 import { isIdentifier, substituteIdentifier, type Identifier } from '@/util/ast/abstract'
 import { partition } from '@/util/data/array'
-import { filterDefined } from '@/util/data/iterable'
 import { Rect } from '@/util/data/rect'
 import { Vec2 } from '@/util/data/vec2'
 import { qnLastSegment, tryQualifiedName } from '@/util/qualifiedName'
 import type { ToValue } from '@/util/reactivity'
+import * as iter from 'enso-common/src/utilities/data/iter'
 import { nextTick, toValue } from 'vue'
 import { assert, assertNever } from 'ydoc-shared/util/assert'
 import { mustExtend } from 'ydoc-shared/util/types'
@@ -76,7 +76,8 @@ export function useNodeCreation(
       : placement.type === 'mouse' ? tryMouse() ?? place()
       : placement.type === 'mouseRelative' ? tryMouseRelative(placement.posOffset) ?? place()
       : placement.type === 'mouseEvent' ? mouseDictatedPlacement(placement.position)
-      : placement.type === 'source' ? place(filterDefined([graphStore.visibleArea(placement.node)]))
+      : placement.type === 'source' ?
+        place(iter.filterDefined([graphStore.visibleArea(placement.node)]))
       : placement.type === 'fixed' ? placement.position
       : assertNever(placement)
     )
@@ -278,7 +279,10 @@ export function insertNodeStatements(
   const lines = bodyBlock.lines
   const lastStatement = lines[lines.length - 1]?.statement?.node
   const index =
-    lastStatement instanceof Ast.MutableAssignment || lastStatement instanceof Ast.MutableFunction ?
+    (
+      lastStatement instanceof Ast.MutableAssignment ||
+      lastStatement instanceof Ast.MutableFunctionDef
+    ) ?
       lines.length
     : lines.length - 1
   bodyBlock.insert(index, ...statements)

app/gui/src/project-view/composables/selection.ts

@@ -3,15 +3,15 @@ import { selectionMouseBindings } from '@/bindings'
 import { useEvent } from '@/composables/events'
 import type { PortId } from '@/providers/portInfo.ts'
 import { type NodeId } from '@/stores/graph'
-import { filter, filterDefined, map } from '@/util/data/iterable'
 import type { Rect } from '@/util/data/rect'
 import { intersectionSize } from '@/util/data/set'
 import { Vec2 } from '@/util/data/vec2'
 import { dataAttribute, elementHierarchy } from '@/util/dom'
+import * as iter from 'enso-common/src/utilities/data/iter'
 import * as set from 'lib0/set'
 import { computed, ref, shallowReactive, shallowRef } from 'vue'
 import { Err, Ok, type Result } from 'ydoc-shared/util/data/result'
-import { NavigatorComposable } from './navigator'
+import type { NavigatorComposable } from './navigator'
 
 interface BaseSelectionOptions<T> {
   margin?: number
@@ -85,11 +85,13 @@ function useSelectionImpl<T, PackedT>(
   // Selection, including elements that do not (currently) pass `isValid`.
   const rawSelected = shallowReactive(new Set<PackedT>())
 
-  const unpackedRawSelected = computed(() => set.from(filterDefined(map(rawSelected, unpack))))
-  const selected = computed(() => set.from(filter(unpackedRawSelected.value, isValid)))
+  const unpackedRawSelected = computed(() =>
+    set.from(iter.filterDefined(iter.map(rawSelected, unpack))),
+  )
+  const selected = computed(() => set.from(iter.filter(unpackedRawSelected.value, isValid)))
   const isChanging = computed(() => anchor.value != null)
   const committedSelection = computed(() =>
-    isChanging.value ? set.from(filter(initiallySelected, isValid)) : selected.value,
+    isChanging.value ? set.from(iter.filter(initiallySelected, isValid)) : selected.value,
   )
 
   function readInitiallySelected() {

app/gui/src/project-view/mock/engine.ts

@@ -1,9 +1,9 @@
+import { Ast } from '@/util/ast'
 import { Pattern } from '@/util/ast/match'
 import type { MockYdocProviderImpl } from '@/util/crdt'
 import type { WebSocketHandler } from '@/util/net'
 import type { QualifiedName } from '@/util/qualifiedName'
 import * as random from 'lib0/random'
-import * as Ast from 'ydoc-shared/ast'
 import {
   Builder,
   EnsoUUID,
@@ -49,6 +49,7 @@ const mainFile = `\
 ## Module documentation
 from Standard.Base import all
 
+## A collapsed function
 func1 arg =
     f2 = Main.func2 arg
     result = f2 - 5

app/gui/src/project-view/providers/fullscreenContext.ts

@@ -1,5 +1,5 @@
 import { createContextStore } from '@/providers'
-import { Ref } from 'vue'
+import type { Ref } from 'vue'
 
 export { provideFn as provideFullscreenContext, injectFn as useFullscreenContext }
 const { provideFn, injectFn } = createContextStore(

app/gui/src/project-view/providers/tooltipState.ts

@@ -1,5 +1,5 @@
 import { createContextStore } from '@/providers'
-import { last } from '@/util/data/iterable'
+import * as iter from 'enso-common/src/utilities/data/iter'
 import {
   computed,
   onUnmounted,
@@ -21,14 +21,14 @@ const { provideFn, injectFn } = createContextStore('tooltip registry', () => {
   const hoveredElements = shallowReactive<Map<HTMLElement, EntriesSet>>(new Map())
 
   const lastHoveredElement = computed(() => {
-    return last(hoveredElements.keys())
+    return iter.last(hoveredElements.keys())
   })
 
   return {
     lastHoveredElement,
     getElementEntry(el: HTMLElement | undefined): TooltipEntry | undefined {
       const set = el && hoveredElements.get(el)
-      return set ? last(set) : undefined
+      return set ? iter.last(set) : undefined
     },
     registerTooltip(slot: Ref<Slot | undefined>) {
       const entry: TooltipEntry = {

app/gui/src/project-view/providers/visualizationConfig.ts

@@ -1,8 +1,8 @@
-import { NodeCreationOptions } from '@/components/GraphEditor/nodeCreation'
-import { ToolbarItem } from '@/components/visualizations/toolbar'
+import type { NodeCreationOptions } from '@/components/GraphEditor/nodeCreation'
+import type { ToolbarItem } from '@/components/visualizations/toolbar'
 import { createContextStore } from '@/providers'
-import { Vec2 } from '@/util/data/vec2'
-import { ToValue } from '@/util/reactivity'
+import type { Vec2 } from '@/util/data/vec2'
+import type { ToValue } from '@/util/reactivity'
 import { reactive } from 'vue'
 
 export interface VisualizationConfig {

app/gui/src/project-view/providers/widgetRegistry.ts

@@ -4,8 +4,7 @@ import type { WidgetConfiguration } from '@/providers/widgetRegistry/configurati
 import type { GraphDb } from '@/stores/graph/graphDatabase'
 import type { Typename } from '@/stores/suggestionDatabase/entry'
 import { Ast } from '@/util/ast'
-import { MutableModule } from '@/util/ast/abstract'
-import { ViteHotContext } from 'vite/types/hot.js'
+import type { ViteHotContext } from 'vite/types/hot.js'
 import { computed, shallowReactive, type Component, type PropType } from 'vue'
 import type { WidgetEditHandlerParent } from './widgetRegistry/editHandler'
 
@@ -169,7 +168,7 @@ export interface WidgetProps<T> {
  * is committed in {@link NodeWidgetTree}.
  */
 export interface WidgetUpdate {
-  edit?: MutableModule | undefined
+  edit?: Ast.MutableModule | undefined
   portUpdate?: { origin: PortId } & (
     | { value: Ast.Owned<Ast.MutableExpression> | string | undefined }
     | { metadataKey: string; metadata: unknown }

app/gui/src/project-view/stores/graph/__tests__/graphDatabase.test.ts

@@ -23,7 +23,7 @@ export function parseWithSpans<T extends Record<string, SourceRange>>(code: stri
     idMap.insertKnownId(span, eid)
   }
 
-  const { root: ast, toRaw, getSpan } = Ast.parseExtended(code, idMap)
+  const { root: ast, toRaw, getSpan } = Ast.parseUpdatingIdMap(code, idMap)
   const idFromExternal = new Map<ExternalId, AstId>()
   ast.visitRecursive((ast) => {
     idFromExternal.set(ast.externalId, ast.id)
@@ -58,7 +58,7 @@ test('Reading graph from definition', () => {
   const db = GraphDb.Mock()
   const expressions = Array.from(ast.statements())
   const func = expressions[0]
-  assert(func instanceof Ast.Function)
+  assert(func instanceof Ast.FunctionDef)
   const rawFunc = toRaw.get(func.id)
   assert(rawFunc?.type === RawAst.Tree.Type.Function)
   db.updateExternalIds(ast)

app/gui/src/project-view/stores/graph/graphDatabase.ts

@@ -23,7 +23,7 @@ import {
 } from '@/util/reactivity'
 import * as objects from 'enso-common/src/utilities/data/object'
 import * as set from 'lib0/set'
-import { reactive, ref, shallowReactive, WatchStopHandle, type Ref } from 'vue'
+import { reactive, ref, shallowReactive, type Ref, type WatchStopHandle } from 'vue'
 import type { MethodCall, StackItem } from 'ydoc-shared/languageServerTypes'
 import type { Opt } from 'ydoc-shared/util/data/opt'
 import type { ExternalId, SourceRange, VisualizationMetadata } from 'ydoc-shared/yjsModel'
@@ -47,7 +47,7 @@ export class BindingsDb {
 
   /** TODO: Add docs */
   readFunctionAst(
-    func: Ast.Function,
+    func: Ast.FunctionDef,
     rawFunc: RawAst.Tree.Function | undefined,
     moduleCode: string,
     getSpan: (id: AstId) => SourceRange | undefined,
@@ -346,7 +346,7 @@ export class GraphDb {
    * expression changes.
    */
   updateNodes(
-    functionAst_: Ast.Function,
+    functionAst_: Ast.FunctionDef,
     { watchEffect }: { watchEffect: (f: () => void) => WatchStopHandle },
   ) {
     const currentNodeIds = new Set<NodeId>()
@@ -468,7 +468,7 @@ export class GraphDb {
 
   /** Deeply scan the function to perform alias-analysis. */
   updateBindings(
-    functionAst_: Ast.Function,
+    functionAst_: Ast.FunctionDef,
     rawFunction: RawAst.Tree.Function | undefined,
     moduleCode: string,
     getSpan: (id: AstId) => SourceRange | undefined,

app/gui/src/project-view/stores/graph/index.ts

@@ -22,15 +22,15 @@ import { isAstId, isIdentifier } from '@/util/ast/abstract'
 import { RawAst, visitRecursive } from '@/util/ast/raw'
 import { reactiveModule } from '@/util/ast/reactive'
 import { partition } from '@/util/data/array'
-import { Events, stringUnionToArray } from '@/util/data/observable'
+import { stringUnionToArray, type Events } from '@/util/data/observable'
 import { Rect } from '@/util/data/rect'
 import { Err, mapOk, Ok, unwrap, type Result } from '@/util/data/result'
 import { Vec2 } from '@/util/data/vec2'
 import { normalizeQualifiedName, qnLastSegment, tryQualifiedName } from '@/util/qualifiedName'
 import { useWatchContext } from '@/util/reactivity'
 import { computedAsync } from '@vueuse/core'
+import * as iter from 'enso-common/src/utilities/data/iter'
 import { map, set } from 'lib0'
-import { iteratorFilter } from 'lib0/iterator'
 import {
   computed,
   markRaw,
@@ -98,7 +98,7 @@ export const { injectFn: useGraphStore, provideFn: provideGraphStore } = createC
     const vizRects = reactive(new Map<NodeId, Rect>())
     // The currently visible nodes' areas (including visualization).
     const visibleNodeAreas = computed(() => {
-      const existing = iteratorFilter(nodeRects.entries(), ([id]) => db.isNodeId(id))
+      const existing = iter.filter(nodeRects.entries(), ([id]) => db.isNodeId(id))
       return Array.from(existing, ([id, rect]) => vizRects.get(id) ?? rect)
     })
     function visibleArea(nodeId: NodeId): Rect | undefined {
@@ -146,7 +146,7 @@ export const { injectFn: useGraphStore, provideFn: provideGraphStore } = createC
       },
     )
 
-    const methodAst = computed<Result<Ast.Function>>(() =>
+    const methodAst = computed<Result<Ast.FunctionDef>>(() =>
       syncModule.value ? getExecutedMethodAst(syncModule.value) : Err('AST not yet initialized'),
     )
 
@@ -188,7 +188,7 @@ export const { injectFn: useGraphStore, provideFn: provideGraphStore } = createC
       db.updateBindings(method, rawFunc, moduleSource.text, getSpan)
     })
 
-    function getExecutedMethodAst(module?: Ast.Module): Result<Ast.Function> {
+    function getExecutedMethodAst(module?: Ast.Module): Result<Ast.FunctionDef> {
       const executionStackTop = proj.executionContext.getStackTop()
       switch (executionStackTop.type) {
         case 'ExplicitCall': {
@@ -204,7 +204,7 @@ export const { injectFn: useGraphStore, provideFn: provideGraphStore } = createC
       }
     }
 
-    function getMethodAst(ptr: MethodPointer, edit?: Ast.Module): Result<Ast.Function> {
+    function getMethodAst(ptr: MethodPointer, edit?: Ast.Module): Result<Ast.FunctionDef> {
       const topLevel = (edit ?? syncModule.value)?.root()
       if (!topLevel) return Err('Module unavailable')
       assert(topLevel instanceof Ast.BodyBlock)

app/gui/src/project-view/stores/graph/unconnectedEdges.ts

@@ -1,7 +1,7 @@
 import type { PortId } from '@/providers/portInfo'
 import type { ConnectedEdge } from '@/stores/graph/index'
-import { filterDefined } from '@/util/data/iterable'
 import { Vec2 } from '@/util/data/vec2'
+import * as iter from 'enso-common/src/utilities/data/iter'
 import { computed, ref, watch, type WatchSource } from 'vue'
 import type { AstId } from 'ydoc-shared/ast'
 
@@ -99,7 +99,7 @@ export function useUnconnectedEdges() {
   const unconnectedEdges = computed<Set<UnconnectedEdge>>(
     () =>
       new Set(
-        filterDefined([mouseEditedEdge.value, cbEditedEdge.value, outputSuggestedEdge.value]),
+        iter.filterDefined([mouseEditedEdge.value, cbEditedEdge.value, outputSuggestedEdge.value]),
       ),
   )
 

app/gui/src/project-view/util/ast/__tests__/abstract.test.ts

@@ -1,8 +1,6 @@
 import { assert, assertDefined } from '@/util/assert'
 import { Ast } from '@/util/ast'
 import {
-  MutableModule,
-  TextLiteral,
   escapeTextLiteral,
   findModuleMethod,
   substituteIdentifier,
@@ -21,7 +19,7 @@ import { findExpressions, testCase, tryFindExpressions } from './testCase'
 test('Raw block abstracts to Ast.BodyBlock', () => {
   const code = 'value = 2 + 2'
   const rawBlock = Ast.rawParseModule(code)
-  const edit = MutableModule.Transient()
+  const edit = Ast.MutableModule.Transient()
   const abstracted = Ast.abstract(edit, rawBlock, code)
   expect(abstracted.root).toBeInstanceOf(Ast.BodyBlock)
 })
@@ -395,7 +393,7 @@ test.each(cases)('parse/print round-trip: %s', (testCase) => {
   const root = Ast.parseModule(code)
   root.module.setRoot(root)
   // Print AST back to source.
-  const printed = Ast.print(root)
+  const printed = Ast.printWithSpans(root)
   expect(printed.code).toEqual(expectedCode)
   // Loading token IDs from IdMaps is not implemented yet, fix during sync.
   printed.info.tokens.clear()
@@ -409,7 +407,7 @@ test.each(cases)('parse/print round-trip: %s', (testCase) => {
   const { root: root1, spans: spans1 } = Ast.parseModuleWithSpans(printed.code)
   Ast.setExternalIds(root1.module, spans1, idMap)
   // Check that Identities match original AST.
-  const printed1 = Ast.print(root1)
+  const printed1 = Ast.printWithSpans(root1)
   printed1.info.tokens.clear()
   const idMap1 = Ast.spanMapToIdMap(printed1.info)
   const mapsEqual = idMap1.isEqual(idMap)
@@ -448,7 +446,7 @@ test('Insert new expression', () => {
 
 type SimpleModule = {
   root: Ast.BodyBlock
-  main: Ast.Function
+  main: Ast.FunctionDef
   mainBlock: Ast.BodyBlock
   assignment: Ast.Assignment
 }
@@ -515,7 +513,7 @@ test('Modify subexpression - setting a vector', () => {
   expect(assignment).toBeInstanceOf(Ast.Assignment)
 
   const edit = root.module.edit()
-  const transientModule = MutableModule.Transient()
+  const transientModule = Ast.MutableModule.Transient()
   const barExpression = Ast.parseExpression('bar')
   assertDefined(barExpression)
   const newValue = Ast.Vector.new(transientModule, [barExpression])
@@ -556,7 +554,7 @@ test('Block lines interface', () => {
 })
 
 test('Splice', () => {
-  const module = MutableModule.Transient()
+  const module = Ast.MutableModule.Transient()
   const edit = module.edit()
   const ident = Ast.Ident.new(edit, 'foo' as Identifier)
   expect(ident.code()).toBe('foo')
@@ -566,7 +564,7 @@ test('Splice', () => {
 })
 
 test('Construct app', () => {
-  const edit = MutableModule.Transient()
+  const edit = Ast.MutableModule.Transient()
   const app = Ast.App.new(
     edit,
     Ast.Ident.new(edit, 'func' as Identifier),
@@ -607,9 +605,9 @@ test('Automatic parenthesis', () => {
 
 test('Tree repair: Non-canonical block line attribution', () => {
   const beforeCase = testCase({
-    'func a b =': Ast.Function,
+    'func a b =': Ast.FunctionDef,
     '    c = a + b': Ast.Assignment,
-    'main =': Ast.Function,
+    'main =': Ast.FunctionDef,
     '    func arg1 arg2': Ast.ExpressionStatement,
   })
   const before = beforeCase.statements
@@ -624,10 +622,12 @@ test('Tree repair: Non-canonical block line attribution', () => {
 
   const repair = edit.edit()
   Ast.repair(editedRoot, repair)
-  const afterRepair = findExpressions(repair.root()!, {
-    'func a b =': Ast.Function,
+  const repairedRoot = repair.root()
+  assertDefined(repairedRoot)
+  const afterRepair = findExpressions(repairedRoot, {
+    'func a b =': Ast.FunctionDef,
     'c = a + b': Ast.Assignment,
-    'main =': Ast.Function,
+    'main =': Ast.FunctionDef,
     'func arg1 arg2': Ast.ExpressionStatement,
   })
   const repairedFunc = afterRepair['func a b =']
@@ -737,7 +737,7 @@ describe('Code edit', () => {
 
   test('Rearrange block', () => {
     const beforeCase = testCase({
-      'main =': Ast.Function,
+      'main =': Ast.FunctionDef,
       '    call_result = func sum 12': Ast.Assignment,
       '    sum = value + 23': Ast.Assignment,
       '    value = 42': Ast.Assignment,
@@ -756,7 +756,7 @@ describe('Code edit', () => {
     expect(edit.root()?.code()).toBe(newCode)
     // Ensure the identities of all the original nodes were maintained.
     const after = tryFindExpressions(edit.root()!, {
-      'main =': Ast.Function,
+      'main =': Ast.FunctionDef,
       'call_result = func sum 12': Ast.Assignment,
       'sum = value + 23': Ast.Assignment,
       'value = 42': Ast.Assignment,
@@ -768,7 +768,7 @@ describe('Code edit', () => {
 
   test('Rename binding', () => {
     const beforeCase = testCase({
-      'main =': Ast.Function,
+      'main =': Ast.FunctionDef,
       '    value = 42': Ast.Assignment,
       '    sum = value + 23': Ast.Assignment,
       '    call_result = func sum 12': Ast.Assignment,
@@ -782,12 +782,14 @@ describe('Code edit', () => {
       '\n    sum = the_number + 23',
       '\n    call_result = func sum 12',
     ].join('')
-    edit.root()!.syncToCode(newCode)
+    const editRoot = edit.root()
+    assertDefined(editRoot)
+    editRoot.syncToCode(newCode)
     // Ensure the change was made.
     expect(edit.root()?.code()).toBe(newCode)
     // Ensure the identities of all the original nodes were maintained.
     const after = tryFindExpressions(edit.root()!, {
-      'main =': Ast.Function,
+      'main =': Ast.FunctionDef,
       'call_result = func sum 12': Ast.Assignment,
       'sum = the_number + 23': Ast.Assignment,
       'the_number = 42': Ast.Assignment,
@@ -840,8 +842,8 @@ describe('Code edit', () => {
   })
 
   test('No-op block change', () => {
-    const code = 'a = 1\nb = 2\n'
-    const block = Ast.parseBlock(code)
+    const code = 'main =\n    a = 1\n    b = 2\n'
+    const block = Ast.parseModule(code)
     const module = block.module
     module.setRoot(block)
     block.syncToCode(code)
@@ -854,7 +856,7 @@ describe('Code edit', () => {
     const before = findExpressions(beforeRoot, {
       value: Ast.Ident,
       '1': Ast.NumericLiteral,
-      'value = 1 +': Ast.Function,
+      'value = 1 +': Ast.FunctionDef,
     })
     const edit = beforeRoot.module.edit()
     const newCode = 'value = 1 \n'
@@ -865,7 +867,7 @@ describe('Code edit', () => {
     const after = findExpressions(edit.root()!, {
       value: Ast.Ident,
       '1': Ast.NumericLiteral,
-      'value = 1': Ast.Function,
+      'value = 1': Ast.FunctionDef,
     })
     expect(after.value.id).toBe(before.value.id)
     expect(after['1'].id).toBe(before['1'].id)
@@ -938,6 +940,7 @@ test.each([
   'Substitute qualified name $pattern inside $original',
   ({ original, pattern, substitution, expected }) => {
     const expression = Ast.parseExpression(original) ?? Ast.parseStatement(original)
+    assertDefined(expression)
     const module = expression.module
     module.setRoot(expression)
     const edit = expression.module.edit()
@@ -994,6 +997,7 @@ test.each([
   'Substitute identifier $pattern inside $original',
   ({ original, pattern, substitution, expected }) => {
     const expression = Ast.parseExpression(original) ?? Ast.parseStatement(original)
+    assertDefined(expression)
     const module = expression.module
     module.setRoot(expression)
     const edit = expression.module.edit()
@@ -1039,7 +1043,7 @@ test.prop({ rawText: sometimesUnicodeString })('Text interpolation roundtrip', (
 })
 
 test.prop({ rawText: sometimesUnicodeString })('AST text literal new', ({ rawText }) => {
-  const literal = TextLiteral.new(rawText)
+  const literal = Ast.TextLiteral.new(rawText)
   expect(literal.rawTextContent).toBe(rawText)
 })
 
@@ -1047,7 +1051,7 @@ test.prop({
   boundary: fc.constantFrom('"', "'"),
   rawText: sometimesUnicodeString,
 })('AST text literal rawTextContent', ({ boundary, rawText }) => {
-  const literal = TextLiteral.new('')
+  const literal = Ast.TextLiteral.new('')
   literal.setBoundaries(boundary)
   literal.setRawTextContent(rawText)
   expect(literal.rawTextContent).toBe(rawText)
@@ -1062,7 +1066,7 @@ test.prop({
 })
 
 test('setRawTextContent promotes single-line uninterpolated text to interpolated if a newline is added', () => {
-  const literal = TextLiteral.new('')
+  const literal = Ast.TextLiteral.new('')
   literal.setBoundaries('"')
   const rawText = '\n'
   literal.setRawTextContent(rawText)
@@ -1093,6 +1097,7 @@ test.each([
   { code: 'operator1 + operator2', expected: { subject: 'operator1 + operator2', accesses: [] } },
 ])('Access chain in $code', ({ code, expected }) => {
   const ast = Ast.parseExpression(code)
+  assertDefined(ast)
   const { subject, accessChain } = Ast.accessChain(ast)
   expect({
     subject: subject.code(),
@@ -1108,7 +1113,9 @@ test.each`
 `('Pushing $pushed to vector $initial', ({ initial, pushed, expected }) => {
   const vector = Ast.Vector.tryParse(initial)
   assertDefined(vector)
-  vector.push(Ast.parseExpression(pushed, vector.module))
+  const elem = Ast.parseExpression(pushed, vector.module)
+  assertDefined(elem)
+  vector.push(elem)
   expect(vector.code()).toBe(expected)
 })
 
@@ -1188,7 +1195,9 @@ test.each`
   ({ initial, index, value, expected }) => {
     const vector = Ast.Vector.tryParse(initial)
     assertDefined(vector)
-    vector.set(index, Ast.parseExpression(value, vector.module))
+    const elemValue = Ast.parseExpression(value, vector.module)
+    assertDefined(elemValue)
+    vector.set(index, elemValue)
     expect(vector.code()).toBe(expected)
   },
 )
@@ -1211,10 +1220,11 @@ test.each`
   ({ ensoNumber, jsNumber, expectedEnsoNumber }) => {
     if (ensoNumber != null) {
       const literal = Ast.parseExpression(ensoNumber)
+      assertDefined(literal)
       expect(tryEnsoToNumber(literal)).toBe(jsNumber)
     }
     if (jsNumber != null) {
-      const convertedToAst = tryNumberToEnso(jsNumber, MutableModule.Transient())
+      const convertedToAst = tryNumberToEnso(jsNumber, Ast.MutableModule.Transient())
       expect(convertedToAst?.code()).toBe(expectedEnsoNumber)
     }
   },

app/gui/src/project-view/util/ast/__tests__/abstractFileIo.test.ts

@@ -7,14 +7,26 @@ import { splitFileContents } from 'ydoc-shared/ensoFile'
 // file format handling to shared and create a test utility for easy *.enso file fixture loading.
 import { deserializeIdMap } from 'ydoc-server'
 
+/** Print the AST and re-parse it, copying `externalId`s (but not other metadata) from the original. */
+function normalize(rootIn: Ast.Ast): Ast.Ast {
+  const printed = Ast.printWithSpans(rootIn)
+  const idMap = Ast.spanMapToIdMap(printed.info)
+  const module = Ast.MutableModule.Transient()
+  const tree = Ast.rawParseModule(printed.code)
+  const { root: parsed, spans } = Ast.abstract(module, tree, printed.code)
+  module.setRoot(parsed)
+  Ast.setExternalIds(module, spans, idMap)
+  return parsed
+}
+
 test('full file IdMap round trip', () => {
   const content = fs.readFileSync(__dirname + '/fixtures/stargazers.enso').toString()
   const { code, idMapJson, metadataJson: _ } = splitFileContents(content)
   const idMapOriginal = deserializeIdMap(idMapJson!)
   const idMap = idMapOriginal.clone()
-  const ast_ = Ast.parseExtended(code, idMapOriginal.clone()).root
-  const ast = Ast.parseExtended(code, idMap).root
-  const ast2 = Ast.normalize(ast)
+  const ast_ = Ast.parseUpdatingIdMap(code, idMapOriginal.clone()).root
+  const ast = Ast.parseUpdatingIdMap(code, idMap).root
+  const ast2 = normalize(ast)
   const astTT = Ast.tokenTreeWithIds(ast)
   expect(ast2.code()).toBe(ast.code())
   expect(Ast.tokenTreeWithIds(ast2), 'Print/parse preserves IDs').toStrictEqual(astTT)

app/gui/src/project-view/util/ast/__tests__/documentation.test.ts

@@ -1,90 +0,0 @@
-import { assert } from '@/util/assert'
-import { Ast } from '@/util/ast'
-import { test } from '@fast-check/vitest'
-import { expect } from 'vitest'
-
-test.each([
-  { code: '## Simple\nnode', documentation: 'Simple' },
-  {
-    code: '## Preferred indent\n   2nd line\n   3rd line\nnode',
-    documentation: 'Preferred indent\n2nd line\n3rd line',
-  },
-  {
-    code: '## Extra-indented child\n 2nd line\n   3rd line\nnode',
-    documentation: 'Extra-indented child\n2nd line\n3rd line',
-    normalized: '## Extra-indented child\n   2nd line\n   3rd line\nnode',
-  },
-  {
-    code: '## Extra-indented child, beyond 4th column\n 2nd line\n        3rd line\nnode',
-    documentation: 'Extra-indented child, beyond 4th column\n2nd line\n    3rd line',
-    normalized: '## Extra-indented child, beyond 4th column\n   2nd line\n       3rd line\nnode',
-  },
-  {
-    code: '##Preferred indent, no initial space\n  2nd line\n  3rd line\nnode',
-    documentation: 'Preferred indent, no initial space\n2nd line\n3rd line',
-    normalized: '## Preferred indent, no initial space\n   2nd line\n   3rd line\nnode',
-  },
-  {
-    code: '## Minimum indent\n 2nd line\n 3rd line\nnode',
-    documentation: 'Minimum indent\n2nd line\n3rd line',
-    normalized: '## Minimum indent\n   2nd line\n   3rd line\nnode',
-  },
-])('Documentation edit round-trip: $code', (docCase) => {
-  const { code, documentation } = docCase
-  const parsed = Ast.parseStatement(code)!
-  const parsedDocumentation = parsed.documentationText()
-  expect(parsedDocumentation).toBe(documentation)
-  const edited = Ast.MutableModule.Transient().copy(parsed)
-  assert('setDocumentationText' in edited)
-  edited.setDocumentationText(parsedDocumentation)
-  expect(edited.code()).toBe(docCase.normalized ?? code)
-})
-
-test.each([
-  '## Some documentation\nf x = 123',
-  '## Some documentation\n    and a second line\nf x = 123',
-  '## Some documentation## Another documentation??\nf x = 123',
-])('Finding documentation: $code', (code) => {
-  const block = Ast.parseBlock(code)
-  const method = Ast.findModuleMethod(block, 'f')!.statement
-  expect(method.documentationText()).toBeTruthy()
-})
-
-test.each([
-  {
-    code: '## Already documented\nf x = 123',
-    expected: '## Already documented\nf x = 123',
-  },
-  {
-    code: 'f x = 123',
-    expected: '##\nf x = 123',
-  },
-])('Adding documentation: $code', ({ code, expected }) => {
-  const block = Ast.parseBlock(code)
-  const module = block.module
-  const method = module.getVersion(Ast.findModuleMethod(block, 'f')!.statement)
-  if (method.documentationText() === undefined) {
-    method.setDocumentationText('')
-  }
-  expect(block.code()).toBe(expected)
-})
-
-test('Creating comments', () => {
-  const block = Ast.parseBlock('2 + 2')
-  block.module.setRoot(block)
-  const statement = [...block.statements()][0]! as Ast.MutableExpressionStatement
-  const docText = 'Calculate five'
-  statement.setDocumentationText(docText)
-  expect(statement.module.root()?.code()).toBe(`## ${docText}\n2 + 2`)
-})
-
-test('Creating comments: indented', () => {
-  const block = Ast.parseBlock('main =\n    x = 1')
-  const module = block.module
-  module.setRoot(block)
-  const main = module.getVersion(Ast.findModuleMethod(block, 'main')!.statement)
-  const statement = [...main.bodyAsBlock().statements()][0]! as Ast.MutableAssignment
-  const docText = 'The smallest natural number'
-  statement.setDocumentationText(docText)
-  expect(statement.module.root()?.code()).toBe(`main =\n    ## ${docText}\n    x = 1`)
-})

app/gui/src/project-view/util/ast/__tests__/raw.test.ts

@@ -1,9 +1,9 @@
 import { RawAst, rawParseModule, readAstOrTokenSpan, walkRecursive } from '@/util/ast/raw'
+import * as iter from 'enso-common/src/utilities/data/iter'
 import { assert, expect, test } from 'vitest'
 import { Token, Tree } from 'ydoc-shared/ast/generated/ast'
 import type { LazyObject } from 'ydoc-shared/ast/parserSupport'
 import { assertDefined } from 'ydoc-shared/util/assert'
-import { tryGetSoleValue } from 'ydoc-shared/util/data/iterable'
 
 /**
  * Read a single line of code
@@ -12,7 +12,7 @@ import { tryGetSoleValue } from 'ydoc-shared/util/data/iterable'
  */
 function rawParseLine(code: string): RawAst.Tree {
   const block = rawParseModule(code)
-  const soleExpression = tryGetSoleValue(block.statements)?.expression
+  const soleExpression = iter.tryGetSoleValue(block.statements)?.expression
   assertDefined(soleExpression)
   return soleExpression
 }

app/gui/src/project-view/util/ast/__tests__/reactive.test.ts

@@ -1,34 +1,40 @@
+import { assert, assertDefined } from '@/util/assert'
 import { Ast } from '@/util/ast'
 import { reactiveModule } from '@/util/ast/reactive'
+import * as iter from 'enso-common/src/utilities/data/iter'
 import { expect, test } from 'vitest'
 import { nextTick, watchEffect } from 'vue'
 import * as Y from 'yjs'
 
+function getAppAtModuleRoot(module: Ast.MutableModule) {
+  const expressionStatement = iter.first(module.root()!.statements())
+  assert(expressionStatement instanceof Ast.MutableExpressionStatement)
+  const app2 = expressionStatement.expression
+  assert(app2 instanceof Ast.MutableApp)
+  return app2
+}
+
 test('Module reactivity: applyEdit', async () => {
-  const beforeEdit = Ast.parseBlock('func arg1 arg2')
+  const beforeEdit = Ast.parseModule('func arg1 arg2')
   beforeEdit.module.setRoot(beforeEdit)
 
   const module = reactiveModule(new Y.Doc(), () => {})
   module.applyEdit(beforeEdit.module)
-  expect(module.root()!.code()).toBe(beforeEdit.code())
+  expect(module.root()?.code()).toBe(beforeEdit.code())
 
-  const app2 = (
-    (module.root() as Ast.MutableBodyBlock).lines[0]!.statement!
-      .node as Ast.MutableExpressionStatement
-  ).expression as unknown as Ast.App
+  const app2 = getAppAtModuleRoot(module)
   let app2Code: string | undefined = undefined
   watchEffect(() => (app2Code = app2.argument.code()))
   expect(app2Code).toBe('arg2')
 
   const edit = beforeEdit.module.edit()
-  const editApp2 = (
-    edit.getVersion(beforeEdit).lines[0]!.statement!.node as Ast.MutableExpressionStatement
-  ).expression as Ast.MutableApp
+  const editApp2 = getAppAtModuleRoot(edit)
   const newArg = Ast.Ident.tryParse('newArg', edit)
+  assertDefined(newArg)
   expect(newArg).toBeDefined()
-  editApp2.setArgument(newArg!)
+  editApp2.setArgument(newArg)
   const codeAfterEdit = 'func arg1 newArg'
-  expect(edit.root()!.code()).toBe(codeAfterEdit)
+  expect(edit.root()?.code()).toBe(codeAfterEdit)
 
   module.applyEdit(edit)
   expect(app2Code).toBe('arg2')
@@ -37,21 +43,21 @@ test('Module reactivity: applyEdit', async () => {
 })
 
 test('Module reactivity: Direct Edit', async () => {
-  const beforeEdit = Ast.parseExpression('func arg1 arg2')
+  const beforeEdit = Ast.parseModule('func arg1 arg2')
   beforeEdit.module.setRoot(beforeEdit)
 
   const module = reactiveModule(new Y.Doc(), () => {})
   module.applyEdit(beforeEdit.module)
-  expect(module.root()!.code()).toBe(beforeEdit.code())
+  expect(module.root()?.code()).toBe(beforeEdit.code())
 
-  const app2 = module.root() as unknown as Ast.MutableApp
+  const app2 = getAppAtModuleRoot(module)
   let app2Code: string | undefined = undefined
   watchEffect(() => (app2Code = app2.argument.code()))
   expect(app2Code).toBe('arg2')
 
   app2.setArgument(Ast.Ident.tryParse('newArg', module)!)
   const codeAfterEdit = 'func arg1 newArg'
-  expect(module.root()!.code()).toBe(codeAfterEdit)
+  expect(module.root()?.code()).toBe(codeAfterEdit)
 
   expect(app2Code).toBe('arg2')
   await nextTick()
@@ -66,14 +72,19 @@ test('Module reactivity: Tracking access to ancestors', async () => {
   module.applyEdit(beforeEdit.module)
   expect(module.root()!.code()).toBe(beforeEdit.code())
 
-  const block = module.root() as any as Ast.BodyBlock
+  const block = module.root()
+  assertDefined(block)
 
-  const [func, otherFunc] = block.statements() as [Ast.Function, Ast.Function]
+  const [func, otherFunc] = block.statements()
+  assert(func instanceof Ast.MutableFunctionDef)
+  assert(otherFunc instanceof Ast.MutableFunctionDef)
   expect(func.name.code()).toBe('main')
   expect(otherFunc.name.code()).toBe('other')
-  const expression = Array.from(func.bodyExpressions())[0]!
+  const expression = iter.first(func.bodyExpressions())
+  assert(!!expression?.isExpression())
   expect(expression.code()).toBe('23')
-  const otherExpression = Array.from(otherFunc.bodyExpressions())[0]!
+  const otherExpression = iter.first(otherFunc.bodyExpressions())
+  assert(!!otherExpression?.isExpression())
   expect(otherExpression.code()).toBe('f')
 
   let parentAccesses = 0
@@ -84,7 +95,9 @@ test('Module reactivity: Tracking access to ancestors', async () => {
   expect(parentAccesses).toBe(1)
 
   const edit = beforeEdit.module.edit()
-  const taken = edit.getVersion(expression).replaceValue(Ast.parseExpression('replacement', edit))
+  const replacementValue = Ast.parseExpression('replacement', edit)
+  assertDefined(replacementValue)
+  const taken = edit.getVersion(expression).replaceValue(replacementValue)
   edit.getVersion(otherExpression).updateValue((oe) => Ast.App.positional(oe, taken, edit))
   module.applyEdit(edit)
 

app/gui/src/project-view/util/ast/abstract.ts

@@ -1,32 +1,41 @@
 import { normalizeQualifiedName, qnFromSegments } from '@/util/qualifiedName'
-import {
-  Ast,
-  BodyBlock,
+import type {
   Expression,
-  Function,
-  Ident,
   IdentifierOrOperatorIdentifier,
   Mutable,
+  MutableExpression,
+  MutableStatement,
+  Owned,
+  QualifiedName,
+  Statement,
+} from 'ydoc-shared/ast'
+import {
+  App,
+  Ast,
+  BodyBlock,
+  FunctionDef,
+  Group,
+  Ident,
   MutableAst,
   MutableBodyBlock,
-  MutableExpression,
-  MutableFunction,
+  MutableFunctionDef,
   MutableIdent,
   MutableModule,
   MutablePropertyAccess,
-  MutableStatement,
   NegationApp,
   NumericLiteral,
   OprApp,
-  Owned,
   PropertyAccess,
-  QualifiedName,
-  Statement,
   Token,
+  Wildcard,
+  abstract,
   isTokenId,
   parseExpression,
-  print,
+  rawParseModule,
+  setExternalIds,
 } from 'ydoc-shared/ast'
+import { spanMapToIdMap, spanMapToSpanGetter } from 'ydoc-shared/ast/idMap'
+import { IdMap } from 'ydoc-shared/yjsModel'
 
 export * from 'ydoc-shared/ast'
 
@@ -39,7 +48,7 @@ export function deserializeExpression(serialized: string): Owned<MutableExpressi
 
 /** Returns a serialized representation of the expression. */
 export function serializeExpression(ast: Expression): string {
-  return print(ast).code
+  return ast.code()
 }
 
 export type TokenTree = (TokenTree | string)[]
@@ -76,7 +85,7 @@ export function tokenTreeWithIds(root: Ast): TokenTree {
 export function moduleMethodNames(topLevel: BodyBlock): Set<string> {
   const result = new Set<string>()
   for (const statement of topLevel.statements()) {
-    if (statement instanceof Function) result.add(statement.name.code())
+    if (statement instanceof FunctionDef) result.add(statement.name.code())
   }
   return result
 }
@@ -84,26 +93,26 @@ export function moduleMethodNames(topLevel: BodyBlock): Set<string> {
 export function findModuleMethod(
   topLevel: MutableBodyBlock,
   name: string,
-): { statement: MutableFunction; index: number } | undefined
+): { statement: MutableFunctionDef; index: number } | undefined
 export function findModuleMethod(
   topLevel: BodyBlock,
   name: string,
-): { statement: Function; index: number } | undefined
+): { statement: FunctionDef; index: number } | undefined
 /** Find the definition of the function with the specified name in the given block. */
 export function findModuleMethod(
   topLevel: BodyBlock,
   name: string,
-): { statement: Function; index: number } | undefined {
+): { statement: FunctionDef; index: number } | undefined {
   // FIXME: We should use alias analysis to handle shadowing correctly.
   const isFunctionWithName = (statement: Statement, name: string) =>
-    statement instanceof Function && statement.name.code() === name
+    statement instanceof FunctionDef && statement.name.code() === name
   const index = topLevel.lines.findIndex(
     (line) => line.statement && isFunctionWithName(line.statement.node, name),
   )
   if (index === -1) return undefined
-  const statement = topLevel.lines[index]!.statement!.node as Function
+  const statement = topLevel.lines[index]!.statement!.node as FunctionDef
   return {
-    /** The `Function` definition. */
+    /** The function definition. */
     statement,
     /** The index into the block's `lines` where the definition was found. */
     index,
@@ -271,6 +280,79 @@ export function dropMutability<T extends Ast>(value: Owned<Mutable<T>>): T {
   return value as unknown as T
 }
 
+function unwrapGroups(ast: Ast) {
+  while (ast instanceof Group && ast.expression) ast = ast.expression
+  return ast
+}
+
+/**
+ * Tries to recognize inputs that are semantically-equivalent to a sequence of `App`s, and returns the arguments
+ * identified and LHS of the analyzable chain.
+ *
+ * In particular, this function currently recognizes syntax used in visualization-preprocessor expressions.
+ */
+export function analyzeAppLike(ast: Ast): { func: Ast; args: Ast[] } {
+  const deferredOperands = new Array<Ast>()
+  while (
+    ast instanceof OprApp &&
+    ast.operator.ok &&
+    ast.operator.value.code() === '<|' &&
+    ast.lhs &&
+    ast.rhs
+  ) {
+    deferredOperands.push(unwrapGroups(ast.rhs))
+    ast = unwrapGroups(ast.lhs)
+  }
+  deferredOperands.reverse()
+  const args = new Array<Ast>()
+  while (ast instanceof App) {
+    const deferredOperand = ast.argument instanceof Wildcard ? deferredOperands.pop() : undefined
+    args.push(deferredOperand ?? unwrapGroups(ast.argument))
+    ast = ast.function
+  }
+  args.reverse()
+  return { func: ast, args }
+}
+
+/**
+ * Unroll the provided chain of `PropertyAccess` nodes, returning the first non-access as `subject` and the accesses
+ * from left-to-right.
+ */
+export function accessChain(ast: Expression): {
+  subject: Expression
+  accessChain: PropertyAccess[]
+} {
+  const accessChain = new Array<PropertyAccess>()
+  while (ast instanceof PropertyAccess && ast.lhs) {
+    accessChain.push(ast)
+    ast = ast.lhs
+  }
+  accessChain.reverse()
+  return { subject: ast, accessChain }
+}
+
+/**
+ * Parse the input, and apply the given `IdMap`. Return the parsed tree, the updated `IdMap`, the span map, and a
+ *  mapping to the `RawAst` representation.
+ */
+export function parseUpdatingIdMap(
+  code: string,
+  idMap?: IdMap | undefined,
+  inModule?: MutableModule,
+) {
+  const rawRoot = rawParseModule(code)
+  const module = inModule ?? MutableModule.Transient()
+  const { root, spans, toRaw } = module.transact(() => {
+    const { root, spans, toRaw } = abstract(module, rawRoot, code)
+    root.module.setRoot(root)
+    if (idMap) setExternalIds(root.module, spans, idMap)
+    return { root, spans, toRaw }
+  })
+  const getSpan = spanMapToSpanGetter(spans)
+  const idMapOut = spanMapToIdMap(spans)
+  return { root, idMap: idMapOut, getSpan, toRaw }
+}
+
 declare const tokenKey: unique symbol
 declare module '@/providers/widgetRegistry' {
   export interface WidgetInputTypes {

app/gui/src/project-view/util/ast/extended.ts

@@ -9,13 +9,13 @@ import {
   walkRecursive,
 } from '@/util/ast/raw'
 import type { Opt } from '@/util/data/opt'
+import * as iter from 'enso-common/src/utilities/data/iter'
 import * as encoding from 'lib0/encoding'
 import * as sha256 from 'lib0/hash/sha256'
 import * as map from 'lib0/map'
 import { markRaw } from 'vue'
 import * as Ast from 'ydoc-shared/ast/generated/ast'
 import { Token, Tree } from 'ydoc-shared/ast/generated/ast'
-import { tryGetSoleValue } from 'ydoc-shared/util/data/iterable'
 import type { ExternalId, IdMap, SourceRange } from 'ydoc-shared/yjsModel'
 
 export { AstExtended as RawAstExtended }
@@ -63,7 +63,7 @@ class AstExtended<T extends Tree | Token = Tree | Token, HasIdMap extends boolea
     const block = AstExtended.parse(code)
     assert(block.isTree(Tree.Type.BodyBlock))
     return block.map((block) => {
-      const soleStatement = tryGetSoleValue(block.statements)
+      const soleStatement = iter.tryGetSoleValue(block.statements)
       assertDefined(soleStatement?.expression)
       return soleStatement.expression
     })

app/gui/src/project-view/util/ast/match.ts

@@ -1,6 +1,6 @@
 import { assert, assertDefined } from '@/util/assert'
 import { Ast } from '@/util/ast'
-import { zipLongest } from '@/util/data/iterable'
+import * as iter from 'enso-common/src/utilities/data/iter'
 
 /**
  * A pattern is an AST object with "placeholder" expressions.
@@ -68,7 +68,7 @@ export class Pattern<T extends Ast.Ast = Ast.Expression> {
   ): Ast.Owned<Ast.Mutable<T>> {
     const template = edit.copy(this.template)
     const placeholders = findPlaceholders(template, this.placeholder).map((ast) => edit.tryGet(ast))
-    for (const [placeholder, replacement] of zipLongest(placeholders, subtrees)) {
+    for (const [placeholder, replacement] of iter.zipLongest(placeholders, subtrees)) {
       assertDefined(placeholder)
       assertDefined(replacement)
       placeholder.replace(replacement)
@@ -118,7 +118,7 @@ function matchSubtree(
       }
     }
   }
-  for (const [patternNode, targetNode] of zipLongest(pattern.children(), target.children())) {
+  for (const [patternNode, targetNode] of iter.zipLongest(pattern.children(), target.children())) {
     if (!patternNode || !targetNode) return false
     if (patternNode instanceof Ast.Token && targetNode instanceof Ast.Token) {
       if (patternNode.code() !== targetNode.code()) return false

app/gui/src/project-view/util/containers.ts

@@ -1,6 +1,6 @@
 import { assertDefined, assertEqual } from '@/util/assert'
 import type { NonEmptyArray } from '@/util/data/array'
-import { mapIterator } from 'lib0/iterator'
+import * as iter from 'enso-common/src/utilities/data/iter'
 
 /**
  * Map that supports Object-based keys.
@@ -85,9 +85,7 @@ export class MappedSet<T extends object> {
     private readonly valueMapper: (key: T) => any,
     elements: Iterable<T> = [],
   ) {
-    this.set = new Map(
-      mapIterator(elements[Symbol.iterator](), (elem) => [valueMapper(elem), elem]),
-    )
+    this.set = new Map(iter.map(elements, (elem) => [valueMapper(elem), elem]))
   }
 
   /** Add the given value to the set. */

app/gui/src/project-view/util/data/iterable.ts

@@ -1,33 +0,0 @@
-export * from 'ydoc-shared/util/data/iterable'
-
-/** TODO: Add docs */
-export function* filterDefined<T>(iterable: Iterable<T | undefined>): IterableIterator<T> {
-  for (const value of iterable) {
-    if (value !== undefined) yield value
-  }
-}
-
-/** TODO: Add docs */
-export function every<T>(iter: Iterable<T>, f: (value: T) => boolean): boolean {
-  for (const value of iter) if (!f(value)) return false
-  return true
-}
-
-/** Return the first element returned by the iterable which meets the condition. */
-export function find<T>(iter: Iterable<T>, f: (value: T) => boolean): T | undefined {
-  for (const value of iter) {
-    if (f(value)) return value
-  }
-  return undefined
-}
-
-/**
- * Return last element returned by the iterable.
- * NOTE: Linear complexity. This function always visits the whole iterable. Using this with an
- * infinite generator will cause an infinite loop.
- */
-export function last<T>(iter: Iterable<T>): T | undefined {
-  let last
-  for (const el of iter) last = el
-  return last
-}

app/gui/src/project-view/util/database/reactiveDb.ts

@@ -11,7 +11,7 @@
  */
 
 import { assert } from '@/util/assert'
-import { LazySyncEffectSet, NonReactiveView } from '@/util/reactivity'
+import { LazySyncEffectSet, type NonReactiveView } from '@/util/reactivity'
 import * as map from 'lib0/map'
 import { ObservableV2 } from 'lib0/observable'
 import * as set from 'lib0/set'

app/gui/src/project-view/util/getIconName.ts

@@ -1,10 +1,7 @@
-import { NodeId } from '@/stores/graph'
+import type { NodeId } from '@/stores/graph'
 import { GraphDb } from '@/stores/graph/graphDatabase'
-import {
-  SuggestionKind,
-  type SuggestionEntry,
-  type Typename,
-} from '@/stores/suggestionDatabase/entry'
+import type { SuggestionEntry, Typename } from '@/stores/suggestionDatabase/entry'
+import { SuggestionKind } from '@/stores/suggestionDatabase/entry'
 import type { Icon } from '@/util/iconName'
 import type { MethodPointer } from 'ydoc-shared/languageServerTypes'
 

app/gui/src/project-view/util/reactivity.ts

@@ -3,29 +3,31 @@
 import { defaultEquality } from '@/util/equals'
 import { debouncedWatch } from '@vueuse/core'
 import { nop } from 'lib0/function'
-import {
-  callWithErrorHandling,
-  computed,
+import type {
   ComputedRef,
   DeepReadonly,
-  effect,
-  effectScope,
-  isRef,
   MaybeRefOrGetter,
-  queuePostFlushCb,
-  reactive,
-  ReactiveEffect,
   ReactiveEffectOptions,
   ReactiveEffectRunner,
   Ref,
+  WatchSource,
+  WatchStopHandle,
+  WritableComputedRef,
+} from 'vue'
+import {
+  callWithErrorHandling,
+  computed,
+  effect,
+  effectScope,
+  isRef,
+  queuePostFlushCb,
+  reactive,
+  ReactiveEffect,
   shallowReactive,
   shallowRef,
   toRaw,
   toValue,
   watch,
-  WatchSource,
-  WatchStopHandle,
-  WritableComputedRef,
 } from 'vue'
 
 /** Cast watch source to an observable ref. */

app/ydoc-server/src/edits.ts

@@ -5,7 +5,7 @@
 
 import diff from 'fast-diff'
 import type { ModuleUpdate } from 'ydoc-shared/ast'
-import { MutableModule, print, spanMapToIdMap } from 'ydoc-shared/ast'
+import { MutableModule, printWithSpans, spanMapToIdMap } from 'ydoc-shared/ast'
 import { EnsoFileParts } from 'ydoc-shared/ensoFile'
 import { TextEdit } from 'ydoc-shared/languageServerTypes'
 import { assert } from 'ydoc-shared/util/assert'
@@ -69,7 +69,7 @@ export function applyDocumentUpdates(
   const root = syncModule.root()
   assert(root != null)
   if (codeChanged || idsChanged || synced.idMapJson == null) {
-    const { code, info } = print(root)
+    const { code, info } = printWithSpans(root)
     if (codeChanged) newCode = code
     newIdMap = spanMapToIdMap(info)
   }

app/ydoc-server/src/languageServerSession.ts

@@ -635,7 +635,7 @@ class ModulePersistence extends ObservableV2<{ removed: () => void }> {
       const astRoot = syncModule.root()
       if (!astRoot) return
       if ((code !== this.syncedCode || idMapJson !== this.syncedIdMap) && idMapJson) {
-        const spans = parsedSpans ?? Ast.print(astRoot).info
+        const spans = parsedSpans ?? Ast.printWithSpans(astRoot).info
         if (idMapJson !== this.syncedIdMap && parsedIdMap === undefined) {
           const idMap = deserializeIdMap(idMapJson)
           const idsAssigned = Ast.setExternalIds(syncModule, spans, idMap)

app/ydoc-shared/package.json

@@ -34,6 +34,7 @@
     }
   },
   "dependencies": {
+    "enso-common": "workspace:*",
     "@noble/hashes": "^1.4.0",
     "@open-rpc/client-js": "^1.8.1",
     "@types/debug": "^4.1.12",

app/ydoc-shared/src/ast/__tests__/documentation.test.ts

@@ -0,0 +1,211 @@
+import { describe, expect, test } from 'vitest'
+import { assert } from '../../util/assert'
+import { MutableModule } from '../mutableModule'
+import { parseBlock, parseModule, parseStatement } from '../parse'
+import { MutableAssignment, MutableExpressionStatement, MutableFunctionDef } from '../tree'
+
+test.each([
+  { code: '## Simple\nnode', documentation: 'Simple' },
+  {
+    code: '## Preferred indent\n   2nd line\n   3rd line\nnode',
+    documentation: 'Preferred indent\n2nd line\n3rd line',
+  },
+  {
+    code: '## Extra-indented child\n 2nd line\n   3rd line\nnode',
+    documentation: 'Extra-indented child\n2nd line\n3rd line',
+    normalized: '## Extra-indented child\n   2nd line\n   3rd line\nnode',
+  },
+  {
+    code: '## Extra-indented child, beyond 4th column\n 2nd line\n        3rd line\nnode',
+    documentation: 'Extra-indented child, beyond 4th column\n2nd line\n    3rd line',
+    normalized: '## Extra-indented child, beyond 4th column\n   2nd line\n       3rd line\nnode',
+  },
+  {
+    code: '##Preferred indent, no initial space\n  2nd line\n  3rd line\nnode',
+    documentation: 'Preferred indent, no initial space\n2nd line\n3rd line',
+    normalized: '## Preferred indent, no initial space\n   2nd line\n   3rd line\nnode',
+  },
+  {
+    code: '## Minimum indent\n 2nd line\n 3rd line\nnode',
+    documentation: 'Minimum indent\n2nd line\n3rd line',
+    normalized: '## Minimum indent\n   2nd line\n   3rd line\nnode',
+  },
+])('Documentation edit round-trip: $code', docCase => {
+  const { code, documentation } = docCase
+  const parsed = parseStatement(code)!
+  const parsedDocumentation = parsed.documentationText()
+  expect(parsedDocumentation).toBe(documentation)
+  const edited = MutableModule.Transient().copy(parsed)
+  assert('setDocumentationText' in edited)
+  edited.setDocumentationText(parsedDocumentation)
+  expect(edited.code()).toBe(docCase.normalized ?? code)
+})
+
+test.each([
+  '## Some documentation\nf x = 123',
+  '## Some documentation\n    and a second line\nf x = 123',
+  '## Some documentation## Another documentation??\nf x = 123',
+])('Finding documentation: $code', code => {
+  const block = parseBlock(code)
+  const method = [...block.statements()][0]
+  assert(method instanceof MutableFunctionDef)
+  expect(method.documentationText()).toBeTruthy()
+})
+
+test.each([
+  {
+    code: '## Already documented\nf x = 123',
+    expected: '## Already documented\nf x = 123',
+  },
+  {
+    code: 'f x = 123',
+    expected: '##\nf x = 123',
+  },
+])('Adding documentation: $code', ({ code, expected }) => {
+  const block = parseBlock(code)
+  const method = [...block.statements()][0]
+  assert(method instanceof MutableFunctionDef)
+  if (method.documentationText() === undefined) method.setDocumentationText('')
+  expect(block.code()).toBe(expected)
+})
+
+test('Creating comments', () => {
+  const block = parseBlock('2 + 2')
+  block.module.setRoot(block)
+  const statement = [...block.statements()][0]
+  assert(statement instanceof MutableExpressionStatement)
+  const docText = 'Calculate five'
+  statement.setDocumentationText(docText)
+  expect(statement.module.root()?.code()).toBe(`## ${docText}\n2 + 2`)
+})
+
+test('Creating comments: indented', () => {
+  const topLevel = parseModule('main =\n    x = 1')
+  topLevel.module.setRoot(topLevel)
+  const main = [...topLevel.statements()][0]
+  assert(main instanceof MutableFunctionDef)
+  const statement = [...main.bodyAsBlock().statements()][0]
+  assert(statement instanceof MutableAssignment)
+  const docText = 'The smallest natural number'
+  statement.setDocumentationText(docText)
+  expect(statement.module.root()?.code()).toBe(`main =\n    ## ${docText}\n    x = 1`)
+})
+
+describe('Markdown documentation', () => {
+  const cases = [
+    {
+      source: '## My function',
+      markdown: 'My function',
+    },
+    {
+      source: '## My function\n\n   Second paragraph',
+      markdown: 'My function\nSecond paragraph',
+    },
+    {
+      source: '## My function\n\n\n   Second paragraph after extra gap',
+      markdown: 'My function\n\nSecond paragraph after extra gap',
+    },
+    {
+      source: '## My function\n   with one hard-wrapped paragraph',
+      markdown: 'My function with one hard-wrapped paragraph',
+      normalized: '## My function with one hard-wrapped paragraph',
+    },
+    {
+      source: '## ICON group\n   My function with an icon',
+      markdown: 'ICON group\nMy function with an icon',
+    },
+    {
+      source: [
+        '## This paragraph is hard-wrapped because it its contents are very very very very long,',
+        'and such long long lines can be inconvenient to work with in most text editors',
+        'because no one likes to scroll horizontally',
+        'but if it is edited the reprinted version will be hard-wrapped differently,',
+        'because apparently someone has gone and wrapped their source code in a manner',
+        'not conforming to the Enso syntax specification',
+        'which requires line length not to exceed 100 characters.',
+      ].join('\n   '),
+      markdown: [
+        'This paragraph is hard-wrapped because it its contents are very very very very long,',
+        'and such long long lines can be inconvenient to work with in most text editors',
+        'because no one likes to scroll horizontally',
+        'but if it is edited the reprinted version will be hard-wrapped differently,',
+        'because apparently someone has gone and wrapped their source code in a manner',
+        'not conforming to the Enso syntax specification',
+        'which requires line length not to exceed 100 characters.',
+      ].join(' '),
+      normalized: [
+        '## This paragraph is hard-wrapped because it its contents are very very very very long, and such',
+        'long long lines can be inconvenient to work with in most text editors because no one likes to',
+        'scroll horizontally but if it is edited the reprinted version will be hard-wrapped differently,',
+        'because apparently someone has gone and wrapped their source code in a manner not conforming to',
+        'the Enso syntax specification which requires line length not to exceed 100 characters.',
+      ].join(' '), // TODO: This should be '\n   ' when hard-wrapping is implemented.
+    },
+  ]
+
+  test.each(cases)('Enso source comments to markdown', ({ source, markdown }) => {
+    const moduleSource = `${source}\nmain =\n    x = 1`
+    const topLevel = parseModule(moduleSource)
+    topLevel.module.setRoot(topLevel)
+    const main = [...topLevel.statements()][0]
+    assert(main instanceof MutableFunctionDef)
+    expect(main.mutableDocumentationMarkdown().toJSON()).toBe(markdown)
+  })
+
+  test.each(cases)('Markdown to Enso source', ({ source, markdown, normalized }) => {
+    const functionCode = 'main =\n    x = 1'
+    const topLevel = parseModule(functionCode)
+    topLevel.module.setRoot(topLevel)
+    const main = [...topLevel.statements()][0]
+    assert(main instanceof MutableFunctionDef)
+    const markdownYText = main.mutableDocumentationMarkdown()
+    expect(markdownYText.toJSON()).toBe('')
+    markdownYText.insert(0, markdown)
+    expect(topLevel.code()).toBe((normalized ?? source) + '\n' + functionCode)
+  })
+
+  test.each(cases)('Unedited comments printed verbatim', ({ source, normalized }) => {
+    if (normalized == null) return
+    const functionCode = `main =\n    x = 1`
+    const moduleSource = source + '\n' + functionCode
+    const topLevel = parseModule(moduleSource)
+    expect(topLevel.code()).not.toBe(normalized + '\n' + functionCode)
+    expect(topLevel.code()).toBe(moduleSource)
+  })
+
+  test.each(cases)('Editing different comments with syncToCode', ({ source }) => {
+    const functionCode = (docs: string) => `${docs}\nmain =\n    x = 1`
+    const moduleOriginalSource = functionCode(source)
+    const topLevel = parseModule(moduleOriginalSource)
+    topLevel.module.setRoot(topLevel)
+    assert(topLevel.code() === moduleOriginalSource)
+    const moduleEditedSource = functionCode('Some new docs')
+    topLevel.syncToCode(moduleEditedSource)
+    expect(topLevel.code()).toBe(moduleEditedSource)
+  })
+
+  test.each(cases)('Setting comments to different content with syncToCode', ({ source }) => {
+    const functionCode = (docs: string) => `${docs}\nmain =\n    x = 1`
+    const moduleOriginalSource = functionCode('## Original docs')
+    const topLevel = parseModule(moduleOriginalSource)
+    const module = topLevel.module
+    module.setRoot(topLevel)
+    assert(module.root()?.code() === moduleOriginalSource)
+    const moduleEditedSource = functionCode(source)
+    module.syncToCode(moduleEditedSource)
+    expect(module.root()?.code()).toBe(moduleEditedSource)
+  })
+
+  test('Setting empty markdown content removes comment', () => {
+    const functionCodeWithoutDocs = `main =\n    x = 1`
+    const originalSourceWithDocComment = '## Some docs\n' + functionCodeWithoutDocs
+    const topLevel = parseModule(originalSourceWithDocComment)
+    expect(topLevel.code()).toBe(originalSourceWithDocComment)
+
+    const main = [...topLevel.statements()][0]
+    assert(main instanceof MutableFunctionDef)
+    const markdownYText = main.mutableDocumentationMarkdown()
+    markdownYText.delete(0, markdownYText.length)
+    expect(topLevel.code()).toBe(functionCodeWithoutDocs)
+  })
+})

app/ydoc-shared/src/ast/__tests__/mutableModule.test.ts

@@ -1,6 +1,6 @@
 import { fc, test } from '@fast-check/vitest'
 import { expect } from 'vitest'
-import { __TEST, isAstId } from '../ast/mutableModule'
+import { __TEST, isAstId } from '../mutableModule'
 
 const { newAstId } = __TEST
 

app/ydoc-shared/src/ast/documentation.ts

@@ -0,0 +1,136 @@
+import { LINE_BOUNDARIES } from 'enso-common/src/utilities/data/string'
+import { xxHash128 } from './ffi'
+import type { ConcreteChild, RawConcreteChild } from './print'
+import { ensureUnspaced, firstChild, preferUnspaced, unspaced } from './print'
+import { Token, TokenType } from './token'
+import type { ConcreteRefs, DeepReadonly, DocLine, TextToken } from './tree'
+
+/** Render a documentation line to concrete tokens. */
+export function* docLineToConcrete(
+  docLine: DeepReadonly<DocLine>,
+  indent: string | null,
+): IterableIterator<RawConcreteChild> {
+  yield firstChild(docLine.docs.open)
+  let prevType = undefined
+  let extraIndent = ''
+  for (const { token } of docLine.docs.elements) {
+    if (token.node.tokenType_ === TokenType.Newline) {
+      yield ensureUnspaced(token, false)
+    } else {
+      if (prevType === TokenType.Newline) {
+        yield { whitespace: indent + extraIndent, node: token.node }
+      } else {
+        if (prevType === undefined) {
+          const leadingSpace = token.node.code_.match(/ */)
+          extraIndent = '  ' + (leadingSpace ? leadingSpace[0] : '')
+        }
+        yield { whitespace: '', node: token.node }
+      }
+    }
+    prevType = token.node.tokenType_
+  }
+  for (const newline of docLine.newlines) yield preferUnspaced(newline)
+}
+
+/**
+ * Render function documentation to concrete tokens. If the `markdown` content has the same value as when `docLine` was
+ * parsed (as indicated by `hash`), the `docLine` will be used (preserving concrete formatting). If it is different, the
+ * `markdown` text will be converted to source tokens.
+ */
+export function functionDocsToConcrete(
+  markdown: string,
+  hash: string | undefined,
+  docLine: DeepReadonly<DocLine> | undefined,
+  indent: string | null,
+): IterableIterator<RawConcreteChild> | undefined {
+  return (
+    hash && docLine && xxHash128(markdown) === hash ? docLineToConcrete(docLine, indent)
+    : markdown ? yTextToTokens(markdown, (indent || '') + '   ')
+    : undefined
+  )
+}
+
+/**
+ * Given Enso documentation comment tokens, returns a model of their Markdown content. This model abstracts away details
+ * such as the locations of line breaks that are not paragraph breaks (e.g. lone newlines denoting hard-wrapping of the
+ * source code).
+ */
+export function abstractMarkdown(elements: undefined | TextToken<ConcreteRefs>[]) {
+  let markdown = ''
+  let newlines = 0
+  let readingTags = true
+  let elidedNewline = false
+  ;(elements ?? []).forEach(({ token: { node } }, i) => {
+    if (node.tokenType_ === TokenType.Newline) {
+      if (readingTags || newlines > 0) {
+        markdown += '\n'
+        elidedNewline = false
+      } else {
+        elidedNewline = true
+      }
+      newlines += 1
+    } else {
+      let nodeCode = node.code()
+      if (i === 0) nodeCode = nodeCode.trimStart()
+      if (elidedNewline) markdown += ' '
+      markdown += nodeCode
+      newlines = 0
+      if (readingTags) {
+        if (!nodeCode.startsWith('ICON ')) {
+          readingTags = false
+        }
+      }
+    }
+  })
+  const hash = xxHash128(markdown)
+  return { markdown, hash }
+}
+
+// TODO: Paragraphs should be hard-wrapped to fit within the column limit, but this requires:
+//  - Recognizing block elements other than paragraphs; we must not split non-paragraph elements.
+//  - Recognizing inline elements; some cannot be split (e.g. links), while some can be broken into two (e.g. bold).
+//    If we break inline elements, we must also combine them when encountered during parsing.
+const ENABLE_INCOMPLETE_WORD_WRAP_SUPPORT = false
+
+function* yTextToTokens(yText: string, indent: string): IterableIterator<ConcreteChild<Token>> {
+  yield unspaced(Token.new('##', TokenType.TextStart))
+  const lines = yText.split(LINE_BOUNDARIES)
+  let printingTags = true
+  for (const [i, value] of lines.entries()) {
+    if (i) {
+      yield unspaced(Token.new('\n', TokenType.Newline))
+      if (value && !printingTags) yield unspaced(Token.new('\n', TokenType.Newline))
+    }
+    printingTags = printingTags && value.startsWith('ICON ')
+    let offset = 0
+    while (offset < value.length) {
+      if (offset !== 0) yield unspaced(Token.new('\n', TokenType.Newline))
+      let wrappedLineEnd = value.length
+      let printableOffset = offset
+      if (i !== 0) {
+        while (printableOffset < value.length && value[printableOffset] === ' ')
+          printableOffset += 1
+      }
+      if (ENABLE_INCOMPLETE_WORD_WRAP_SUPPORT && !printingTags) {
+        const ENSO_SOURCE_MAX_COLUMNS = 100
+        const MIN_DOC_COLUMNS = 40
+        const availableWidth = Math.max(
+          ENSO_SOURCE_MAX_COLUMNS - indent.length - (i === 0 && offset === 0 ? '## '.length : 0),
+          MIN_DOC_COLUMNS,
+        )
+        if (availableWidth < wrappedLineEnd - printableOffset) {
+          const wrapIndex = value.lastIndexOf(' ', printableOffset + availableWidth)
+          if (printableOffset < wrapIndex) {
+            wrappedLineEnd = wrapIndex
+          }
+        }
+      }
+      while (printableOffset < value.length && value[printableOffset] === ' ') printableOffset += 1
+      const whitespace = i === 0 && offset === 0 ? ' ' : indent
+      const wrappedLine = value.substring(printableOffset, wrappedLineEnd)
+      yield { whitespace, node: Token.new(wrappedLine, TokenType.TextSection) }
+      offset = wrappedLineEnd
+    }
+  }
+  yield unspaced(Token.new('\n', TokenType.Newline))
+}

app/ydoc-shared/src/ast/idMap.ts

@@ -0,0 +1,64 @@
+import * as random from 'lib0/random'
+import { assert } from '../util/assert'
+import type { ExternalId, SourceRange, SourceRangeKey } from '../yjsModel'
+import { IdMap, isUuid, sourceRangeFromKey, sourceRangeKey } from '../yjsModel'
+import type { Token } from './token'
+import type { Ast, AstId } from './tree'
+
+declare const nodeKeyBrand: unique symbol
+/** A source-range key for an `Ast`. */
+export type NodeKey = SourceRangeKey & { [nodeKeyBrand]: never }
+declare const tokenKeyBrand: unique symbol
+/** A source-range key for a `Token`. */
+export type TokenKey = SourceRangeKey & { [tokenKeyBrand]: never }
+/** Create a source-range key for an `Ast`. */
+export function nodeKey(start: number, length: number): NodeKey {
+  return sourceRangeKey([start, start + length]) as NodeKey
+}
+/** Create a source-range key for a `Token`. */
+export function tokenKey(start: number, length: number): TokenKey {
+  return sourceRangeKey([start, start + length]) as TokenKey
+}
+
+/** Maps from source ranges to `Ast`s. */
+export type NodeSpanMap = Map<NodeKey, Ast[]>
+/** Maps from source ranges to `Token`s. */
+export type TokenSpanMap = Map<TokenKey, Token>
+
+/** Maps from source ranges to `Ast`s and `Token`s. */
+export interface SpanMap {
+  nodes: NodeSpanMap
+  tokens: TokenSpanMap
+}
+
+/** Create a new random {@link ExternalId}. */
+export function newExternalId(): ExternalId {
+  return random.uuidv4() as ExternalId
+}
+
+/** Generate an `IdMap` from a `SpanMap`. */
+export function spanMapToIdMap(spans: SpanMap): IdMap {
+  const idMap = new IdMap()
+  for (const [key, token] of spans.tokens.entries()) {
+    assert(isUuid(token.id))
+    idMap.insertKnownId(sourceRangeFromKey(key), token.id)
+  }
+  for (const [key, asts] of spans.nodes.entries()) {
+    for (const ast of asts) {
+      assert(isUuid(ast.externalId))
+      idMap.insertKnownId(sourceRangeFromKey(key), ast.externalId)
+    }
+  }
+  return idMap
+}
+
+/** Given a `SpanMap`, return a function that can look up source ranges by AST ID. */
+export function spanMapToSpanGetter(spans: SpanMap): (id: AstId) => SourceRange | undefined {
+  const reverseMap = new Map<AstId, SourceRange>()
+  for (const [key, asts] of spans.nodes) {
+    for (const ast of asts) {
+      reverseMap.set(ast.id, sourceRangeFromKey(key))
+    }
+  }
+  return id => reverseMap.get(id)
+}

app/ydoc-shared/src/ast/index.ts

@@ -1,46 +1,16 @@
-import * as random from 'lib0/random'
 import { reachable } from '../util/data/graph'
-import type { ExternalId } from '../yjsModel'
 import type { Module } from './mutableModule'
-import type { SyncTokenId } from './token'
-import type { AstId, MutableAst } from './tree'
-import { App, Ast, Group, OprApp, Wildcard } from './tree'
+import type { AstId } from './tree'
 
+export { spanMapToIdMap } from './idMap'
 export * from './mutableModule'
 export * from './parse'
+export { printWithSpans } from './print'
+export { repair } from './repair'
 export * from './text'
 export * from './token'
 export * from './tree'
 
-declare const brandOwned: unique symbol
-/**
- * Used to mark references required to be unique.
- *
- *  Note that the typesystem cannot stop you from copying an `Owned`,
- *  but that is an easy mistake to see (because it occurs locally).
- *
- *  We can at least require *obtaining* an `Owned`,
- *  which statically prevents the otherwise most likely usage errors when rearranging ASTs.
- */
-export type Owned<T = MutableAst> = T & { [brandOwned]: never }
-/** @internal */
-export function asOwned<T>(t: T): Owned<T> {
-  return t as Owned<T>
-}
-
-export type NodeChild<T> = { whitespace: string | undefined; node: T }
-export type RawNodeChild = NodeChild<AstId> | NodeChild<SyncTokenId>
-
-/** Create a new random {@link ExternalId}. */
-export function newExternalId(): ExternalId {
-  return random.uuidv4() as ExternalId
-}
-
-/** @internal */
-export function parentId(ast: Ast): AstId | undefined {
-  return ast.fields.get('parent')
-}
-
 /** Returns the given IDs, and the IDs of all their ancestors. */
 export function subtrees(module: Module, ids: Iterable<AstId>) {
   return reachable(ids, id => {
@@ -68,37 +38,3 @@ export function subtreeRoots(module: Module, ids: Set<AstId>): Set<AstId> {
   }
   return roots
 }
-
-function unwrapGroups(ast: Ast) {
-  while (ast instanceof Group && ast.expression) ast = ast.expression
-  return ast
-}
-
-/**
- * Tries to recognize inputs that are semantically-equivalent to a sequence of `App`s, and returns the arguments
- *  identified and LHS of the analyzable chain.
- *
- *  In particular, this function currently recognizes syntax used in visualization-preprocessor expressions.
- */
-export function analyzeAppLike(ast: Ast): { func: Ast; args: Ast[] } {
-  const deferredOperands = new Array<Ast>()
-  while (
-    ast instanceof OprApp &&
-    ast.operator.ok &&
-    ast.operator.value.code() === '<|' &&
-    ast.lhs &&
-    ast.rhs
-  ) {
-    deferredOperands.push(unwrapGroups(ast.rhs))
-    ast = unwrapGroups(ast.lhs)
-  }
-  deferredOperands.reverse()
-  const args = new Array<Ast>()
-  while (ast instanceof App) {
-    const deferredOperand = ast.argument instanceof Wildcard ? deferredOperands.pop() : undefined
-    args.push(deferredOperand ?? unwrapGroups(ast.argument))
-    ast = ast.function
-  }
-  args.reverse()
-  return { func: ast, args }
-}

app/ydoc-shared/src/ast/mutableModule.ts

@@ -1,28 +1,30 @@
 import * as random from 'lib0/random'
 import * as Y from 'yjs'
-import {
-  AstId,
-  MutableBodyBlock,
-  NodeChild,
-  Owned,
-  RawNodeChild,
-  SyncTokenId,
-  Token,
-  asOwned,
-  isTokenId,
-  newExternalId,
-  parseModule,
-  subtreeRoots,
-} from '.'
+import { subtreeRoots } from '.'
 import { assert, assertDefined } from '../util/assert'
 import type { SourceRangeEdit } from '../util/data/text'
 import { defaultLocalOrigin, tryAsOrigin, type ExternalId, type Origin } from '../yjsModel'
-import type { AstFields, FixedMap, Mutable } from './tree'
+import { newExternalId } from './idMap'
+import { parseModule } from './parse'
+import type { SyncTokenId } from './token'
+import { Token, isTokenId } from './token'
+import type {
+  AstFields,
+  AstId,
+  BodyBlock,
+  FixedMap,
+  Mutable,
+  MutableAst,
+  MutableBodyBlock,
+  MutableInvalid,
+  NodeChild,
+  Owned,
+  RawNodeChild,
+} from './tree'
 import {
   Ast,
-  MutableAst,
-  MutableInvalid,
   Wildcard,
+  asOwned,
   composeFieldData,
   invalidFields,
   materializeMutable,
@@ -31,7 +33,7 @@ import {
 
 export interface Module {
   edit(): MutableModule
-  root(): Ast | undefined
+  root(): BodyBlock | undefined
   tryGet(id: AstId | undefined): Ast | undefined
 
   /////////////////////////////////
@@ -98,8 +100,8 @@ export class MutableModule implements Module {
   }
 
   /** Return the top-level block of the module. */
-  root(): MutableAst | undefined {
-    return this.rootPointer()?.expression
+  root(): MutableBodyBlock | undefined {
+    return this.rootPointer()?.expression as MutableBodyBlock | undefined
   }
 
   /** Set the given block to be the top-level block of the module. */
@@ -271,24 +273,34 @@ export class MutableModule implements Module {
           node.get(key as any),
         ])
         updateBuilder.updateFields(id, changes)
-      } else if (event.target.parent.parent === this.nodes) {
-        // Updates to fields of a metadata object within a node.
+      } else if (event.target.parent?.parent === this.nodes) {
+        // Updates to fields of an object within a node.
         const id = event.target.parent.get('id')
         DEV: assertAstId(id)
         const node = this.nodes.get(id)
         if (!node) continue
-        const metadata = node.get('metadata') as unknown as Map<string, unknown>
-        const changes: (readonly [string, unknown])[] = Array.from(event.changes.keys, ([key]) => [
-          key,
-          metadata.get(key as any),
-        ])
-        updateBuilder.updateMetadata(id, changes)
-      } else if (event.target.parent.parent.parent === this.nodes) {
-        // Updates to some specific widget's metadata
+        const metadata = node.get('metadata')
+        if (event.target === metadata) {
+          const changes: (readonly [string, unknown])[] = Array.from(
+            event.changes.keys,
+            ([key]) => [key, metadata.get(key as any)],
+          )
+          updateBuilder.updateMetadata(id, changes)
+        } else {
+          // `AbstractType` in node fields.
+          updateBuilder.nodesUpdated.add(id)
+        }
+      } else if (event.target.parent?.parent?.parent === this.nodes) {
         const id = event.target.parent.parent.get('id')
         assertAstId(id)
-        if (!this.nodes.get(id)) continue
-        updateBuilder.updateWidgets(id)
+        const node = this.nodes.get(id)
+        if (!node) continue
+        const metadata = node.get('metadata')
+        const widgets = metadata?.get('widget')
+        if (event.target === widgets) {
+          // Updates to some specific widget's metadata
+          updateBuilder.updateWidgets(id)
+        }
       }
     }
     return updateBuilder.finish()
@@ -475,7 +487,6 @@ class UpdateBuilder {
         assert(value instanceof Y.Map)
         metadataChanges = new Map<string, unknown>(value.entries())
       } else {
-        assert(!(value instanceof Y.AbstractType))
         fieldsChanged = true
       }
     }

app/ydoc-shared/src/ast/parse.ts

@@ -1,70 +1,49 @@
+import * as iter from 'enso-common/src/utilities/data/iter'
 import * as map from 'lib0/map'
-import {
+import * as Y from 'yjs'
+import { assert } from '../util/assert'
+import type { IdMap } from '../yjsModel'
+import { abstractMarkdown } from './documentation'
+import { parse_block, parse_module } from './ffi'
+import * as RawAst from './generated/ast'
+import type { NodeKey, NodeSpanMap, SpanMap, TokenSpanMap } from './idMap'
+import { nodeKey, tokenKey } from './idMap'
+import { MutableModule } from './mutableModule'
+import type { LazyObject } from './parserSupport'
+import { Token } from './token'
+import type {
+  Ast,
   AstId,
-  FunctionFields,
-  Module,
-  MutableInvalid,
+  FunctionDefFields,
+  MutableBodyBlock,
+  MutableExpression,
+  MutableStatement,
   NodeChild,
   Owned,
   OwnedRefs,
   TextElement,
   TextToken,
-  Token,
-  asOwned,
-  isTokenId,
-  newExternalId,
-  parentId,
-  rewriteRefs,
-  subtreeRoots,
-  syncFields,
-  syncNodeMetadata,
-} from '.'
-import { assert, assertDefined, assertEqual } from '../util/assert'
-import { tryGetSoleValue, zip } from '../util/data/iterable'
-import type { SourceRangeEdit, SpanTree } from '../util/data/text'
-import {
-  applyTextEdits,
-  applyTextEditsToSpans,
-  enclosingSpans,
-  textChangeToEdits,
-  trimEnd,
-} from '../util/data/text'
-import {
-  IdMap,
-  isUuid,
-  rangeLength,
-  sourceRangeFromKey,
-  sourceRangeKey,
-  type SourceRange,
-  type SourceRangeKey,
-} from '../yjsModel'
-import { parse_block, parse_module, xxHash128 } from './ffi'
-import * as RawAst from './generated/ast'
-import { MutableModule } from './mutableModule'
-import type { LazyObject } from './parserSupport'
+} from './tree'
 import {
   App,
+  asOwned,
   Assignment,
-  Ast,
   AutoscopedIdentifier,
   BodyBlock,
   ExpressionStatement,
-  Function,
+  FunctionDef,
   Generic,
   Group,
   Ident,
   Import,
   Invalid,
-  MutableAssignment,
-  MutableAst,
-  MutableBodyBlock,
-  MutableExpression,
   MutableExpressionStatement,
   MutableIdent,
-  MutableStatement,
+  MutableInvalid,
   NegationApp,
   NumericLiteral,
   OprApp,
+  parentId,
   PropertyAccess,
   TextLiteral,
   UnaryOprApp,
@@ -89,18 +68,6 @@ function deserializeBlock(blob: Uint8Array): RawAst.Tree.BodyBlock {
   return tree
 }
 
-/** Print the AST and re-parse it, copying `externalId`s (but not other metadata) from the original. */
-export function normalize(rootIn: Ast): Ast {
-  const printed = print(rootIn)
-  const idMap = spanMapToIdMap(printed.info)
-  const module = MutableModule.Transient()
-  const tree = rawParseModule(printed.code)
-  const { root: parsed, spans } = abstract(module, tree, printed.code)
-  module.setRoot(parsed)
-  setExternalIds(module, spans, idMap)
-  return parsed
-}
-
 /** Produce `Ast` types from `RawAst` parser output. */
 export function abstract(
   module: MutableModule,
@@ -247,7 +214,7 @@ class Abstractor {
             [this.abstractToken(tree.opr.value)]
           : Array.from(tree.opr.error.payload.operators, this.abstractToken.bind(this))
         const rhs = tree.rhs ? this.abstractExpression(tree.rhs) : undefined
-        const soleOpr = tryGetSoleValue(opr)
+        const soleOpr = iter.tryGetSoleValue(opr)
         if (soleOpr?.node.code() === '.' && rhs?.node instanceof MutableIdent) {
           // Propagate type.
           const rhs_ = { ...rhs, node: rhs.node }
@@ -350,6 +317,9 @@ class Abstractor {
 
   private abstractFunction(tree: RawAst.Tree.Function) {
     const docLine = tree.docLine && this.abstractDocLine(tree.docLine)
+    const { markdown: docMarkdown, hash: docLineMarkdownHash } = abstractMarkdown(
+      docLine?.docs.elements,
+    )
     const annotationLines = Array.from(tree.annotationLines, anno => ({
       annotation: {
         operator: this.abstractToken(anno.annotation.operator),
@@ -382,8 +352,10 @@ class Abstractor {
     }))
     const equals = this.abstractToken(tree.equals)
     const body = tree.body !== undefined ? this.abstractExpression(tree.body) : undefined
-    return Function.concrete(this.module, {
+    return FunctionDef.concrete(this.module, {
       docLine,
+      docLineMarkdownHash,
+      docMarkdown: new Y.Text(docMarkdown),
       annotationLines,
       signatureLine,
       private_,
@@ -391,7 +363,7 @@ class Abstractor {
       argumentDefinitions,
       equals,
       body,
-    } satisfies FunctionFields<OwnedRefs>)
+    } satisfies FunctionDefFields<OwnedRefs>)
   }
 
   private abstractToken(token: RawAst.Token): { whitespace: string; node: Token } {
@@ -477,143 +449,6 @@ class Abstractor {
   }
 }
 
-declare const nodeKeyBrand: unique symbol
-/** A source-range key for an `Ast`. */
-export type NodeKey = SourceRangeKey & { [nodeKeyBrand]: never }
-declare const tokenKeyBrand: unique symbol
-/** A source-range key for a `Token`. */
-export type TokenKey = SourceRangeKey & { [tokenKeyBrand]: never }
-/** Create a source-range key for an `Ast`. */
-export function nodeKey(start: number, length: number): NodeKey {
-  return sourceRangeKey([start, start + length]) as NodeKey
-}
-/** Create a source-range key for a `Token`. */
-export function tokenKey(start: number, length: number): TokenKey {
-  return sourceRangeKey([start, start + length]) as TokenKey
-}
-
-/** Maps from source ranges to `Ast`s. */
-export type NodeSpanMap = Map<NodeKey, Ast[]>
-/** Maps from source ranges to `Token`s. */
-export type TokenSpanMap = Map<TokenKey, Token>
-
-/** Maps from source ranges to `Ast`s and `Token`s. */
-export interface SpanMap {
-  nodes: NodeSpanMap
-  tokens: TokenSpanMap
-}
-
-/** Code with an associated mapping to `Ast` types. */
-interface PrintedSource {
-  info: SpanMap
-  code: string
-}
-
-/** Generate an `IdMap` from a `SpanMap`. */
-export function spanMapToIdMap(spans: SpanMap): IdMap {
-  const idMap = new IdMap()
-  for (const [key, token] of spans.tokens.entries()) {
-    assert(isUuid(token.id))
-    idMap.insertKnownId(sourceRangeFromKey(key), token.id)
-  }
-  for (const [key, asts] of spans.nodes.entries()) {
-    for (const ast of asts) {
-      assert(isUuid(ast.externalId))
-      idMap.insertKnownId(sourceRangeFromKey(key), ast.externalId)
-    }
-  }
-  return idMap
-}
-
-/** Given a `SpanMap`, return a function that can look up source ranges by AST ID. */
-export function spanMapToSpanGetter(spans: SpanMap): (id: AstId) => SourceRange | undefined {
-  const reverseMap = new Map<AstId, SourceRange>()
-  for (const [key, asts] of spans.nodes) {
-    for (const ast of asts) {
-      reverseMap.set(ast.id, sourceRangeFromKey(key))
-    }
-  }
-  return id => reverseMap.get(id)
-}
-
-/** Return stringification with associated ID map. This is only exported for testing. */
-export function print(ast: Ast): PrintedSource {
-  const info: SpanMap = {
-    nodes: new Map(),
-    tokens: new Map(),
-  }
-  const code = ast.printSubtree(info, 0, null)
-  return { info, code }
-}
-
-/**
- * Used by `Ast.printSubtree`.
- * @internal
- */
-export function printAst(
-  ast: Ast,
-  info: SpanMap,
-  offset: number,
-  parentIndent: string | null,
-  verbatim: boolean = false,
-): string {
-  let code = ''
-  let currentLineIndent = parentIndent
-  let prevIsNewline = false
-  let isFirstToken = offset === 0
-  for (const child of ast.concreteChildren({ verbatim, indent: parentIndent })) {
-    if (!isTokenId(child.node) && ast.module.get(child.node) === undefined) continue
-    if (prevIsNewline) currentLineIndent = child.whitespace
-    const token = isTokenId(child.node) ? ast.module.getToken(child.node) : undefined
-    // Every line in a block starts with a newline token. In an AST produced by the parser, the newline token at the
-    // first line of a module is zero-length. In order to handle whitespace correctly if the lines of a module are
-    // rearranged, if a zero-length newline is encountered within a block, it is printed as an ordinary newline
-    // character, and if an ordinary newline is found at the beginning of the output, it is not printed; however if the
-    // output begins with a newline including a (plain) comment, we print the line as we would in any other block.
-    if (
-      token?.tokenType_ == RawAst.Token.Type.Newline &&
-      isFirstToken &&
-      (!token.code_ || token.code_ === '\n')
-    ) {
-      prevIsNewline = true
-      isFirstToken = false
-      continue
-    }
-    code += child.whitespace
-    if (token) {
-      const tokenStart = offset + code.length
-      prevIsNewline = token.tokenType_ == RawAst.Token.Type.Newline
-      let tokenCode = token.code_
-      if (token.tokenType_ == RawAst.Token.Type.Newline) {
-        tokenCode = tokenCode || '\n'
-      }
-      const span = tokenKey(tokenStart, tokenCode.length)
-      info.tokens.set(span, token)
-      code += tokenCode
-    } else {
-      assert(!isTokenId(child.node))
-      prevIsNewline = false
-      const childNode = ast.module.get(child.node)
-      code += childNode.printSubtree(info, offset + code.length, currentLineIndent, verbatim)
-      // Extra structural validation.
-      assertEqual(childNode.id, child.node)
-      if (parentId(childNode) !== ast.id) {
-        console.error(`Inconsistent parent pointer (expected ${ast.id})`, childNode)
-      }
-      assertEqual(parentId(childNode), ast.id)
-    }
-    isFirstToken = false
-  }
-  // Adjustment to handle an edge case: A module starts with a zero-length newline token. If its first line is indented,
-  // the initial whitespace belongs to the first line because it isn't hoisted past the (zero-length) newline to be the
-  // leading whitespace for the block. In that case, our representation of the block contains leading whitespace at the
-  // beginning, which must be excluded when calculating spans.
-  const leadingWhitespace = code.match(/ */)?.[0].length ?? 0
-  const span = nodeKey(offset + leadingWhitespace, code.length - leadingWhitespace)
-  map.setIfUndefined(info.nodes, span, (): Ast[] => []).unshift(ast)
-  return code
-}
-
 /** Parse the input as a complete module. */
 export function parseModule(code: string, module?: MutableModule): Owned<MutableBodyBlock> {
   return parseModuleWithSpans(code, module).root
@@ -635,7 +470,7 @@ export function parseStatement(
 ): Owned<MutableStatement> | undefined {
   const module_ = module ?? MutableModule.Transient()
   const ast = parseBlock(code, module)
-  const soleStatement = tryGetSoleValue(ast.statements())
+  const soleStatement = iter.tryGetSoleValue(ast.statements())
   if (!soleStatement) return
   const parent = parentId(soleStatement)
   if (parent) module_.delete(parent)
@@ -653,7 +488,7 @@ export function parseExpression(
 ): Owned<MutableExpression> | undefined {
   const module_ = module ?? MutableModule.Transient()
   const ast = parseBlock(code, module)
-  const soleStatement = tryGetSoleValue(ast.statements())
+  const soleStatement = iter.tryGetSoleValue(ast.statements())
   if (!(soleStatement instanceof MutableExpressionStatement)) return undefined
   const expression = soleStatement.expression
   module_.delete(soleStatement.id)
@@ -672,24 +507,6 @@ export function parseModuleWithSpans(
   return abstract(module ?? MutableModule.Transient(), tree, code)
 }
 
-/**
- * Parse the input, and apply the given `IdMap`. Return the parsed tree, the updated `IdMap`, the span map, and a
- *  mapping to the `RawAst` representation.
- */
-export function parseExtended(code: string, idMap?: IdMap | undefined, inModule?: MutableModule) {
-  const rawRoot = rawParseModule(code)
-  const module = inModule ?? MutableModule.Transient()
-  const { root, spans, toRaw } = module.transact(() => {
-    const { root, spans, toRaw } = abstract(module, rawRoot, code)
-    root.module.setRoot(root)
-    if (idMap) setExternalIds(root.module, spans, idMap)
-    return { root, spans, toRaw }
-  })
-  const getSpan = spanMapToSpanGetter(spans)
-  const idMapOut = spanMapToIdMap(spans)
-  return { root, idMap: idMapOut, getSpan, toRaw }
-}
-
 /** Return the number of `Ast`s in the tree, including the provided root. */
 export function astCount(ast: Ast): number {
   let count = 0
@@ -717,363 +534,3 @@ export function setExternalIds(edit: MutableModule, spans: SpanMap, ids: IdMap):
   }
   return astsMatched
 }
-
-/**
- * Try to find all the spans in `expected` in `encountered`. If any are missing, use the provided `code` to determine
- *  whether the lost spans are single-line or multi-line.
- */
-function checkSpans(expected: NodeSpanMap, encountered: NodeSpanMap, code: string) {
-  const lost = new Array<readonly [NodeKey, Ast]>()
-  for (const [key, asts] of expected) {
-    const outermostPrinted = asts[0]
-    if (!outermostPrinted) continue
-    for (let i = 1; i < asts.length; ++i) assertEqual(asts[i]?.parentId, asts[i - 1]?.id)
-    const encounteredAsts = encountered.get(key)
-    if (encounteredAsts === undefined) lost.push([key, outermostPrinted])
-  }
-  const lostInline = new Array<Ast>()
-  const lostBlock = new Array<Ast>()
-  for (const [key, ast] of lost) {
-    const [start, end] = sourceRangeFromKey(key)
-    // Do not report lost empty body blocks, we don't want them to be considered for repair.
-    if (start === end && ast instanceof BodyBlock) continue
-    ;(code.substring(start, end).match(/[\r\n]/) ? lostBlock : lostInline).push(ast)
-  }
-  return { lostInline, lostBlock }
-}
-
-/**
- * If the input tree's concrete syntax has precedence errors (i.e. its expected code would not parse back to the same
- *  structure), try to fix it. If possible, it will be repaired by inserting parentheses; if that doesn't fix it, the
- *  affected subtree will be re-synced to faithfully represent the source code the incorrect tree prints to.
- */
-export function repair(
-  root: BodyBlock,
-  module?: MutableModule,
-): { code: string; fixes: MutableModule | undefined } {
-  // Print the input to see what spans its nodes expect to have in the output.
-  const printed = print(root)
-  // Parse the printed output to see what spans actually correspond to nodes in the printed code.
-  const reparsed = parseModuleWithSpans(printed.code)
-  // See if any span we expected to be a node isn't; if so, it likely merged with its parent due to wrong precedence.
-  const { lostInline, lostBlock } = checkSpans(
-    printed.info.nodes,
-    reparsed.spans.nodes,
-    printed.code,
-  )
-  if (lostInline.length === 0) {
-    if (lostBlock.length !== 0) {
-      console.warn(`repair: Bad block elements, but all inline elements OK?`)
-      const fixes = module ?? root.module.edit()
-      resync(lostBlock, printed.info.nodes, reparsed.spans.nodes, fixes)
-      return { code: printed.code, fixes }
-    }
-    return { code: printed.code, fixes: undefined }
-  }
-
-  // Wrap any "lost" nodes in parentheses.
-  const fixes = module ?? root.module.edit()
-  for (const ast of lostInline) {
-    if (ast instanceof Group) continue
-    fixes.getVersion(ast).update(ast => Group.new(fixes, ast as any))
-  }
-
-  // Verify that it's fixed.
-  const printed2 = print(fixes.root()!)
-  const reparsed2 = parseModuleWithSpans(printed2.code)
-  const { lostInline: lostInline2, lostBlock: lostBlock2 } = checkSpans(
-    printed2.info.nodes,
-    reparsed2.spans.nodes,
-    printed2.code,
-  )
-  if (lostInline2.length !== 0 || lostBlock2.length !== 0)
-    resync([...lostInline2, ...lostBlock2], printed2.info.nodes, reparsed2.spans.nodes, fixes)
-
-  return { code: printed2.code, fixes }
-}
-
-/**
- * Replace subtrees in the module to ensure that the module contents are consistent with the module's code.
- * @param badAsts - ASTs that, if printed, would not parse to exactly their current content.
- * @param badSpans - Span map produced by printing the `badAsts` nodes and all their parents.
- * @param goodSpans - Span map produced by parsing the code from the module of `badAsts`.
- * @param edit - Module to apply the fixes to; must contain all ASTs in `badAsts`.
- */
-function resync(
-  badAsts: Iterable<Ast>,
-  badSpans: NodeSpanMap,
-  goodSpans: NodeSpanMap,
-  edit: MutableModule,
-) {
-  const parentsOfBadSubtrees = new Set<AstId>()
-  const badAstIds = new Set(Array.from(badAsts, ast => ast.id))
-  for (const id of subtreeRoots(edit, badAstIds)) {
-    const parent = edit.get(id)?.parentId
-    if (parent) parentsOfBadSubtrees.add(parent)
-  }
-
-  const spanOfBadParent = new Array<readonly [AstId, NodeKey]>()
-  for (const [span, asts] of badSpans) {
-    for (const ast of asts) {
-      if (parentsOfBadSubtrees.has(ast.id)) spanOfBadParent.push([ast.id, span])
-    }
-  }
-  // All ASTs in the module of badAsts should have entries in badSpans.
-  assertEqual(spanOfBadParent.length, parentsOfBadSubtrees.size)
-
-  for (const [id, span] of spanOfBadParent) {
-    const parent = edit.get(id)
-    const goodAst = goodSpans.get(span)?.[0]
-    // The parent of the root of a bad subtree must be a good AST.
-    assertDefined(goodAst)
-    parent.syncToCode(goodAst.code())
-  }
-
-  console.warn(
-    `repair: Replaced ${parentsOfBadSubtrees.size} subtrees with their reparsed equivalents.`,
-    parentsOfBadSubtrees,
-  )
-}
-
-/**
- * Recursion helper for {@link syntaxHash}.
- * @internal
- */
-function hashSubtreeSyntax(ast: Ast, hashesOut: Map<SyntaxHash, Ast[]>): SyntaxHash {
-  let content = ''
-  content += ast.typeName + ':'
-  for (const child of ast.concreteChildren({ verbatim: false, indent: '' })) {
-    content += child.whitespace ?? '?'
-    if (isTokenId(child.node)) {
-      content += 'Token:' + hashString(ast.module.getToken(child.node).code())
-    } else {
-      content += hashSubtreeSyntax(ast.module.get(child.node), hashesOut)
-    }
-  }
-  const astHash = hashString(content)
-  map.setIfUndefined(hashesOut, astHash, (): Ast[] => []).unshift(ast)
-  return astHash
-}
-
-declare const brandHash: unique symbol
-/** See {@link syntaxHash}. */
-type SyntaxHash = string & { [brandHash]: never }
-/** Applies the syntax-data hashing function to the input, and brands the result as a `SyntaxHash`. */
-function hashString(input: string): SyntaxHash {
-  return xxHash128(input) as SyntaxHash
-}
-
-/**
- * Calculates `SyntaxHash`es for the given node and all its children.
- *
- *  Each `SyntaxHash` summarizes the syntactic content of an AST. If two ASTs have the same code and were parsed the
- *  same way (i.e. one was not parsed in a context that resulted in a different interpretation), they will have the same
- *  hash. Note that the hash is invariant to metadata, including `externalId` assignments.
- */
-function syntaxHash(root: Ast) {
-  const hashes = new Map<SyntaxHash, Ast[]>()
-  const rootHash = hashSubtreeSyntax(root, hashes)
-  return { root: rootHash, hashes }
-}
-
-/** Update `ast` to match the given source code, while modifying it as little as possible. */
-export function syncToCode(ast: MutableAst, code: string, metadataSource?: Module) {
-  const codeBefore = ast.code()
-  const textEdits = textChangeToEdits(codeBefore, code)
-  applyTextEditsToAst(ast, textEdits, metadataSource ?? ast.module)
-}
-
-/** Find nodes in the input `ast` that should be treated as equivalents of nodes in `parsedRoot`. */
-function calculateCorrespondence(
-  ast: Ast,
-  astSpans: NodeSpanMap,
-  parsedRoot: Ast,
-  parsedSpans: NodeSpanMap,
-  textEdits: SourceRangeEdit[],
-  codeAfter: string,
-): Map<AstId, Ast> {
-  const newSpans = new Map<AstId, SourceRange>()
-  for (const [key, asts] of parsedSpans) {
-    for (const ast of asts) newSpans.set(ast.id, sourceRangeFromKey(key))
-  }
-
-  // Retained-code matching: For each new tree, check for some old tree of the same type such that the new tree is the
-  // smallest node to contain all characters of the old tree's code that were not deleted in the edit.
-  //
-  // If the new node's span exactly matches the retained code, add the match to `toSync`. If the new node's span
-  // contains additional code, add the match to `candidates`.
-  const toSync = new Map<AstId, Ast>()
-  const candidates = new Map<AstId, Ast>()
-  const allSpansBefore = Array.from(astSpans.keys(), sourceRangeFromKey)
-  const spansBeforeAndAfter = applyTextEditsToSpans(textEdits, allSpansBefore).map(
-    ([before, after]) => [before, trimEnd(after, codeAfter)] satisfies [any, any],
-  )
-  const partAfterToAstBefore = new Map<SourceRangeKey, Ast>()
-  for (const [spanBefore, partAfter] of spansBeforeAndAfter) {
-    const astBefore = astSpans.get(sourceRangeKey(spanBefore) as NodeKey)![0]!
-    partAfterToAstBefore.set(sourceRangeKey(partAfter), astBefore)
-  }
-  const matchingPartsAfter = spansBeforeAndAfter.map(([_before, after]) => after)
-  const parsedSpanTree = new AstWithSpans(parsedRoot, id => newSpans.get(id)!)
-  const astsMatchingPartsAfter = enclosingSpans(parsedSpanTree, matchingPartsAfter)
-  for (const [astAfter, partsAfter] of astsMatchingPartsAfter) {
-    for (const partAfter of partsAfter) {
-      const astBefore = partAfterToAstBefore.get(sourceRangeKey(partAfter))!
-      if (astBefore.typeName() === astAfter.typeName()) {
-        ;(rangeLength(newSpans.get(astAfter.id)!) === rangeLength(partAfter) ?
-          toSync
-        : candidates
-        ).set(astBefore.id, astAfter)
-        break
-      }
-    }
-  }
-
-  // Index the matched nodes.
-  const oldIdsMatched = new Set<AstId>()
-  const newIdsMatched = new Set<AstId>()
-  for (const [oldId, newAst] of toSync) {
-    oldIdsMatched.add(oldId)
-    newIdsMatched.add(newAst.id)
-  }
-
-  // Movement matching: For each new tree that hasn't been matched, match it with any identical unmatched old tree.
-  const newHashes = syntaxHash(parsedRoot).hashes
-  const oldHashes = syntaxHash(ast).hashes
-  for (const [hash, newAsts] of newHashes) {
-    const unmatchedNewAsts = newAsts.filter(ast => !newIdsMatched.has(ast.id))
-    const unmatchedOldAsts = oldHashes.get(hash)?.filter(ast => !oldIdsMatched.has(ast.id)) ?? []
-    for (const [unmatchedNew, unmatchedOld] of zip(unmatchedNewAsts, unmatchedOldAsts)) {
-      toSync.set(unmatchedOld.id, unmatchedNew)
-      // Update the matched-IDs indices.
-      oldIdsMatched.add(unmatchedOld.id)
-      newIdsMatched.add(unmatchedNew.id)
-    }
-  }
-
-  // Apply any non-optimal span matches from `candidates`, if the nodes involved were not matched during
-  // movement-matching.
-  for (const [beforeId, after] of candidates) {
-    if (oldIdsMatched.has(beforeId) || newIdsMatched.has(after.id)) continue
-    toSync.set(beforeId, after)
-  }
-
-  return toSync
-}
-
-/** Update `ast` according to changes to its corresponding source code. */
-export function applyTextEditsToAst(
-  ast: MutableAst,
-  textEdits: SourceRangeEdit[],
-  metadataSource: Module,
-) {
-  const printed = print(ast)
-  const code = applyTextEdits(printed.code, textEdits)
-  const astModuleRoot = ast.module.root()
-  const rawParsedBlock =
-    ast instanceof MutableBodyBlock && astModuleRoot && ast.is(astModuleRoot) ?
-      rawParseModule(code)
-    : rawParseBlock(code)
-  const rawParsedStatement =
-    ast instanceof MutableBodyBlock ? undefined : (
-      tryGetSoleValue(rawParsedBlock.statements)?.expression
-    )
-  const rawParsedExpression =
-    ast.isExpression() ?
-      rawParsedStatement?.type === RawAst.Tree.Type.ExpressionStatement ?
-        rawParsedStatement.expression
-      : undefined
-    : undefined
-  const rawParsed = rawParsedExpression ?? rawParsedStatement ?? rawParsedBlock
-  const parsed = abstract(ast.module, rawParsed, code)
-  const toSync = calculateCorrespondence(
-    ast,
-    printed.info.nodes,
-    parsed.root,
-    parsed.spans.nodes,
-    textEdits,
-    code,
-  )
-  syncTree(ast, parsed.root, toSync, ast.module, metadataSource)
-}
-
-/** Replace `target` with `newContent`, reusing nodes according to the correspondence in `toSync`. */
-function syncTree(
-  target: Ast,
-  newContent: Owned,
-  toSync: Map<AstId, Ast>,
-  edit: MutableModule,
-  metadataSource: Module,
-) {
-  const newIdToEquivalent = new Map<AstId, AstId>()
-  for (const [beforeId, after] of toSync) newIdToEquivalent.set(after.id, beforeId)
-  const childReplacerFor = (parentId: AstId) => (id: AstId) => {
-    const original = newIdToEquivalent.get(id)
-    if (original) {
-      const replacement = edit.get(original)
-      if (replacement.parentId !== parentId) replacement.fields.set('parent', parentId)
-      return original
-    } else {
-      const child = edit.get(id)
-      if (child.parentId !== parentId) child.fields.set('parent', parentId)
-    }
-  }
-  const parentId = target.fields.get('parent')
-  assertDefined(parentId)
-  const parent = edit.get(parentId)
-  const targetSyncEquivalent = toSync.get(target.id)
-  const syncRoot = targetSyncEquivalent?.id === newContent.id ? targetSyncEquivalent : undefined
-  if (!syncRoot) {
-    parent.replaceChild(target.id, newContent)
-    newContent.fields.set('metadata', target.fields.get('metadata').clone())
-    target.fields.get('metadata').set('externalId', newExternalId())
-  }
-  const newRoot = syncRoot ? target : newContent
-  newRoot.visitRecursive(ast => {
-    const syncFieldsFrom = toSync.get(ast.id)
-    const editAst = edit.getVersion(ast)
-    if (syncFieldsFrom) {
-      const originalAssignmentExpression =
-        ast instanceof Assignment ?
-          metadataSource.get(ast.fields.get('expression').node)
-        : undefined
-      syncFields(edit.getVersion(ast), syncFieldsFrom, childReplacerFor(ast.id))
-      if (editAst instanceof MutableAssignment && originalAssignmentExpression) {
-        if (editAst.expression.externalId !== originalAssignmentExpression.externalId)
-          editAst.expression.setExternalId(originalAssignmentExpression.externalId)
-        syncNodeMetadata(
-          editAst.expression.mutableNodeMetadata(),
-          originalAssignmentExpression.nodeMetadata,
-        )
-      }
-    } else {
-      rewriteRefs(editAst, childReplacerFor(ast.id))
-    }
-    return true
-  })
-}
-
-/** Provides a `SpanTree` view of an `Ast`, given span information. */
-class AstWithSpans implements SpanTree<Ast> {
-  private readonly ast: Ast
-  private readonly getSpan: (astId: AstId) => SourceRange
-
-  constructor(ast: Ast, getSpan: (astId: AstId) => SourceRange) {
-    this.ast = ast
-    this.getSpan = getSpan
-  }
-
-  id(): Ast {
-    return this.ast
-  }
-
-  span(): SourceRange {
-    return this.getSpan(this.ast.id)
-  }
-
-  *children(): IterableIterator<SpanTree<Ast>> {
-    for (const child of this.ast.children()) {
-      if (child instanceof Ast) yield new AstWithSpans(child, this.getSpan)
-    }
-  }
-}

app/ydoc-shared/src/ast/print.ts

@@ -0,0 +1,191 @@
+import * as map from 'lib0/map'
+import { assert, assertEqual } from '../util/assert'
+import type { SpanMap } from './idMap'
+import { nodeKey, tokenKey } from './idMap'
+import type { SyncTokenId } from './token'
+import { isTokenId, TokenType } from './token'
+import type { Ast, AstId } from './tree'
+import { parentId } from './tree'
+
+/** An AST node of a given type with fully-specified whitespace. */
+export interface ConcreteChild<T> {
+  whitespace: string
+  node: T
+}
+/** An AST node in raw (serialization) form, with fully-specified whitespace. */
+export type RawConcreteChild = ConcreteChild<AstId> | ConcreteChild<SyncTokenId>
+
+export function spaced<T extends object | string>(node: T): ConcreteChild<T>
+export function spaced<T extends object | string>(node: T | undefined): ConcreteChild<T> | undefined
+/** @returns The input with leading whitespace specified to be a single space. */
+export function spaced<T extends object | string>(
+  node: T | undefined,
+): ConcreteChild<T> | undefined {
+  if (node === undefined) return node
+  return { whitespace: ' ', node }
+}
+
+export function unspaced<T extends object | string>(node: T): ConcreteChild<T>
+export function unspaced<T extends object | string>(
+  node: T | undefined,
+): ConcreteChild<T> | undefined
+/** @returns The input with leading whitespace specified to be absent. */
+export function unspaced<T extends object | string>(
+  node: T | undefined,
+): ConcreteChild<T> | undefined {
+  if (node === undefined) return node
+  return { whitespace: '', node }
+}
+
+export interface OptionalWhitespace {
+  whitespace?: string | undefined
+}
+export interface WithWhitespace extends OptionalWhitespace {
+  whitespace: string
+}
+/** @returns The input with leading whitespace as specified. */
+export function withWhitespace<T>(node: T, whitespace: string): T & WithWhitespace {
+  return { ...node, whitespace }
+}
+
+/** @returns The input with leading whitespace set to a single space or an empty string, depending on the provided boolean. */
+export function ensureSpacedOnlyIf<T extends OptionalWhitespace>(
+  child: T,
+  condition: boolean,
+  verbatim: boolean | undefined,
+): T & WithWhitespace {
+  return condition ? ensureSpaced(child, verbatim) : ensureUnspaced(child, verbatim)
+}
+
+/** @returns Whether the input value has whitespace specified (as opposed to autospaced). */
+export function isConcrete<T extends OptionalWhitespace>(child: T): child is T & WithWhitespace {
+  return child.whitespace !== undefined
+}
+/** @returns The input if it satisfies {@link isConcrete} */
+export function tryAsConcrete<T extends OptionalWhitespace>(
+  child: T,
+): (T & WithWhitespace) | undefined {
+  return isConcrete(child) ? child : undefined
+}
+/** @returns The input with leading whitespace specified to be a single space, unless it is specified as a non-autospaced value and `verbatim` is `true`. */
+export function ensureSpaced<T extends OptionalWhitespace>(
+  child: T,
+  verbatim: boolean | undefined,
+): T & WithWhitespace {
+  const concreteInput = tryAsConcrete(child)
+  if (verbatim && concreteInput) return concreteInput
+  return concreteInput?.whitespace ? concreteInput : { ...child, whitespace: ' ' }
+}
+/** @returns The input with leading whitespace specified to be empty, unless it is specified as a non-autospaced value and `verbatim` is `true`. */
+export function ensureUnspaced<T extends OptionalWhitespace>(
+  child: T,
+  verbatim: boolean | undefined,
+): T & WithWhitespace {
+  const concreteInput = tryAsConcrete(child)
+  if (verbatim && concreteInput) return concreteInput
+  return concreteInput?.whitespace === '' ? concreteInput : { ...child, whitespace: '' }
+}
+/** @returns The input with leading whitespace specified to be empty. This is equivalent to other ways of clearing the whitespace, such as {@link unspaced}, but using this function documents the reason. */
+export function firstChild<T extends OptionalWhitespace>(child: T): T & WithWhitespace {
+  const concreteInput = tryAsConcrete(child)
+  return concreteInput?.whitespace === '' ? concreteInput : { ...child, whitespace: '' }
+}
+/** If the input is autospaced, returns it with spacing according to the provided `condition`; otherwise, returns it with its existing spacing. */
+export function preferSpacedIf<T extends OptionalWhitespace>(
+  child: T,
+  condition: boolean,
+): T & WithWhitespace {
+  return condition ? preferSpaced(child) : preferUnspaced(child)
+}
+/** If the input is autospaced, returns it unspaced; otherwise, returns it with its existing spacing. */
+export function preferUnspaced<T extends OptionalWhitespace>(child: T): T & WithWhitespace {
+  return tryAsConcrete(child) ?? { ...child, whitespace: '' }
+}
+/** If the input is autospaced, returns it spaced; otherwise, returns it with its existing spacing. */
+export function preferSpaced<T extends OptionalWhitespace>(child: T): T & WithWhitespace {
+  return tryAsConcrete(child) ?? { ...child, whitespace: ' ' }
+}
+
+/** Code with an associated mapping to `Ast` types. */
+interface PrintedSource {
+  info: SpanMap
+  code: string
+}
+
+/** Return stringification with associated ID map. This is only exported for testing. */
+export function printWithSpans(ast: Ast): PrintedSource {
+  const info: SpanMap = {
+    nodes: new Map(),
+    tokens: new Map(),
+  }
+  const code = ast.printSubtree(info, 0, null)
+  return { info, code }
+}
+
+/**
+ * Implementation of `Ast.printSubtree`.
+ * @internal
+ */
+export function printAst(
+  ast: Ast,
+  info: SpanMap,
+  offset: number,
+  parentIndent: string | null,
+  verbatim: boolean = false,
+): string {
+  let code = ''
+  let currentLineIndent = parentIndent
+  let prevIsNewline = false
+  let isFirstToken = offset === 0
+  for (const child of ast.concreteChildren({ verbatim, indent: parentIndent })) {
+    if (!isTokenId(child.node) && ast.module.get(child.node) === undefined) continue
+    if (prevIsNewline) currentLineIndent = child.whitespace
+    const token = isTokenId(child.node) ? ast.module.getToken(child.node) : undefined
+    // Every line in a block starts with a newline token. In an AST produced by the parser, the newline token at the
+    // first line of a module is zero-length. In order to handle whitespace correctly if the lines of a module are
+    // rearranged, if a zero-length newline is encountered within a block, it is printed as an ordinary newline
+    // character, and if an ordinary newline is found at the beginning of the output, it is not printed; however if the
+    // output begins with a newline including a (plain) comment, we print the line as we would in any other block.
+    if (
+      token?.tokenType_ == TokenType.Newline &&
+      isFirstToken &&
+      (!token.code_ || token.code_ === '\n')
+    ) {
+      prevIsNewline = true
+      isFirstToken = false
+      continue
+    }
+    code += child.whitespace
+    if (token) {
+      const tokenStart = offset + code.length
+      prevIsNewline = token.tokenType_ == TokenType.Newline
+      let tokenCode = token.code_
+      if (token.tokenType_ == TokenType.Newline) {
+        tokenCode = tokenCode || '\n'
+      }
+      const span = tokenKey(tokenStart, tokenCode.length)
+      info.tokens.set(span, token)
+      code += tokenCode
+    } else {
+      assert(!isTokenId(child.node))
+      prevIsNewline = false
+      const childNode = ast.module.get(child.node)
+      code += childNode.printSubtree(info, offset + code.length, currentLineIndent, verbatim)
+      // Extra structural validation.
+      assertEqual(childNode.id, child.node)
+      if (parentId(childNode) !== ast.id) {
+        console.error(`Inconsistent parent pointer (expected ${ast.id})`, childNode)
+      }
+      assertEqual(parentId(childNode), ast.id)
+    }
+    isFirstToken = false
+  }
+  // Adjustment to handle an edge case: A module starts with a zero-length newline token. If its first line is indented,
+  // the initial whitespace belongs to the first line because it isn't hoisted past the (zero-length) newline to be the
+  // leading whitespace for the block. In that case, our representation of the block contains leading whitespace at the
+  // beginning, which must be excluded when calculating spans.
+  const leadingWhitespace = code.match(/ */)?.[0].length ?? 0
+  const span = nodeKey(offset + leadingWhitespace, code.length - leadingWhitespace)
+  map.setIfUndefined(info.nodes, span, (): Ast[] => []).unshift(ast)
+  return code
+}

app/ydoc-shared/src/ast/repair.ts

@@ -0,0 +1,126 @@
+import { subtreeRoots } from '.'
+import { assertDefined, assertEqual } from '../util/assert'
+import { sourceRangeFromKey } from '../yjsModel'
+import type { NodeKey, NodeSpanMap } from './idMap'
+import type { MutableModule } from './mutableModule'
+import { parseModuleWithSpans } from './parse'
+import { printWithSpans } from './print'
+import type { Ast, AstId } from './tree'
+import { BodyBlock, Group } from './tree'
+
+/**
+ * Try to find all the spans in `expected` in `encountered`. If any are missing, use the provided `code` to determine
+ *  whether the lost spans are single-line or multi-line.
+ */
+function checkSpans(expected: NodeSpanMap, encountered: NodeSpanMap, code: string) {
+  const lost = new Array<readonly [NodeKey, Ast]>()
+  for (const [key, asts] of expected) {
+    const outermostPrinted = asts[0]
+    if (!outermostPrinted) continue
+    for (let i = 1; i < asts.length; ++i) assertEqual(asts[i]?.parentId, asts[i - 1]?.id)
+    const encounteredAsts = encountered.get(key)
+    if (encounteredAsts === undefined) lost.push([key, outermostPrinted])
+  }
+  const lostInline = new Array<Ast>()
+  const lostBlock = new Array<Ast>()
+  for (const [key, ast] of lost) {
+    const [start, end] = sourceRangeFromKey(key)
+    // Do not report lost empty body blocks, we don't want them to be considered for repair.
+    if (start === end && ast instanceof BodyBlock) continue
+    ;(code.substring(start, end).match(/[\r\n]/) ? lostBlock : lostInline).push(ast)
+  }
+  return { lostInline, lostBlock }
+}
+
+/**
+ * If the input tree's concrete syntax has precedence errors (i.e. its expected code would not parse back to the same
+ *  structure), try to fix it. If possible, it will be repaired by inserting parentheses; if that doesn't fix it, the
+ *  affected subtree will be re-synced to faithfully represent the source code the incorrect tree prints to.
+ */
+export function repair(
+  root: BodyBlock,
+  module?: MutableModule,
+): { code: string; fixes: MutableModule | undefined } {
+  // Print the input to see what spans its nodes expect to have in the output.
+  const printed = printWithSpans(root)
+  // Parse the printed output to see what spans actually correspond to nodes in the printed code.
+  const reparsed = parseModuleWithSpans(printed.code)
+  // See if any span we expected to be a node isn't; if so, it likely merged with its parent due to wrong precedence.
+  const { lostInline, lostBlock } = checkSpans(
+    printed.info.nodes,
+    reparsed.spans.nodes,
+    printed.code,
+  )
+  if (lostInline.length === 0) {
+    if (lostBlock.length !== 0) {
+      console.warn(`repair: Bad block elements, but all inline elements OK?`)
+      const fixes = module ?? root.module.edit()
+      resync(lostBlock, printed.info.nodes, reparsed.spans.nodes, fixes)
+      return { code: printed.code, fixes }
+    }
+    return { code: printed.code, fixes: undefined }
+  }
+
+  // Wrap any "lost" nodes in parentheses.
+  const fixes = module ?? root.module.edit()
+  for (const ast of lostInline) {
+    if (ast instanceof Group) continue
+    fixes.getVersion(ast).update(ast => Group.new(fixes, ast as any))
+  }
+
+  // Verify that it's fixed.
+  const printed2 = printWithSpans(fixes.root()!)
+  const reparsed2 = parseModuleWithSpans(printed2.code)
+  const { lostInline: lostInline2, lostBlock: lostBlock2 } = checkSpans(
+    printed2.info.nodes,
+    reparsed2.spans.nodes,
+    printed2.code,
+  )
+  if (lostInline2.length !== 0 || lostBlock2.length !== 0)
+    resync([...lostInline2, ...lostBlock2], printed2.info.nodes, reparsed2.spans.nodes, fixes)
+
+  return { code: printed2.code, fixes }
+}
+
+/**
+ * Replace subtrees in the module to ensure that the module contents are consistent with the module's code.
+ * @param badAsts - ASTs that, if printed, would not parse to exactly their current content.
+ * @param badSpans - Span map produced by printing the `badAsts` nodes and all their parents.
+ * @param goodSpans - Span map produced by parsing the code from the module of `badAsts`.
+ * @param edit - Module to apply the fixes to; must contain all ASTs in `badAsts`.
+ */
+function resync(
+  badAsts: Iterable<Ast>,
+  badSpans: NodeSpanMap,
+  goodSpans: NodeSpanMap,
+  edit: MutableModule,
+) {
+  const parentsOfBadSubtrees = new Set<AstId>()
+  const badAstIds = new Set(Array.from(badAsts, ast => ast.id))
+  for (const id of subtreeRoots(edit, badAstIds)) {
+    const parent = edit.get(id)?.parentId
+    if (parent) parentsOfBadSubtrees.add(parent)
+  }
+
+  const spanOfBadParent = new Array<readonly [AstId, NodeKey]>()
+  for (const [span, asts] of badSpans) {
+    for (const ast of asts) {
+      if (parentsOfBadSubtrees.has(ast.id)) spanOfBadParent.push([ast.id, span])
+    }
+  }
+  // All ASTs in the module of badAsts should have entries in badSpans.
+  assertEqual(spanOfBadParent.length, parentsOfBadSubtrees.size)
+
+  for (const [id, span] of spanOfBadParent) {
+    const parent = edit.get(id)
+    const goodAst = goodSpans.get(span)?.[0]
+    // The parent of the root of a bad subtree must be a good AST.
+    assertDefined(goodAst)
+    parent.syncToCode(goodAst.code())
+  }
+
+  console.warn(
+    `repair: Replaced ${parentsOfBadSubtrees.size} subtrees with their reparsed equivalents.`,
+    parentsOfBadSubtrees,
+  )
+}