[GUI2] Fix and add tests for delta format translation. (#7968)

Fixes #7967

The text updates should no longer be rejected after applying edits containing newlines. The tricky update translation logic was also covered with property-based unit tests.

https://github.com/enso-org/enso/assets/919491/0bfb6181-7244-4eff-8d72-5b1a4630b9a6

app/gui2/eslint.config.js

@@ -20,7 +20,13 @@ const conf = [
       parserOptions: {
         tsconfigRootDir: DIR_NAME,
         ecmaVersion: 'latest',
-        project: ['./tsconfig.app.json', './tsconfig.node.json', './tsconfig.vitest.json'],
+        project: [
+          './tsconfig.app.json',
+          './tsconfig.node.json',
+          './tsconfig.server.json',
+          './tsconfig.app.vitest.json',
+          './tsconfig.server.vitest.json',
+        ],
       },
     },
     rules: {

app/gui2/package.json

@@ -16,7 +16,7 @@
     "test": "vitest run",
     "build-only": "vite build",
     "compile-server": "tsc -p tsconfig.server.json",
-    "typecheck": "vue-tsc --noEmit -p tsconfig.vitest.json --composite false",
+    "typecheck": "vue-tsc --noEmit -p tsconfig.app.json --composite false",
     "lint": "eslint .",
     "format": "prettier --write src/ && eslint . --fix",
     "build-rust-ffi": "cd rust-ffi && wasm-pack build --release --target web",
@@ -24,6 +24,7 @@
   },
   "dependencies": {
     "@babel/parser": "^7.22.16",
+    "@fast-check/vitest": "^0.0.8",
     "@open-rpc/client-js": "^1.8.1",
     "@vueuse/core": "^10.4.1",
     "codemirror": "^6.0.1",

app/gui2/src/components/CodeEditor.vue

@@ -17,7 +17,6 @@ const shown = ref(false)
 const rootElement = ref<HTMLElement>()
 
 useWindowEvent('keydown', (e) => {
-  console.log('keydown', e)
   const graphEditorInFocus = document.activeElement === document.body
   const codeEditorInFocus = rootElement.value?.contains(document.activeElement)
   const validFocus = graphEditorInFocus || codeEditorInFocus

app/gui2/tsconfig.app.json

@@ -9,7 +9,7 @@
     "public/**/*",
     "public/**/*.vue"
   ],
-  "exclude": ["src/**/__tests__/*"],
+  "exclude": ["src/**/__tests__/*", "shared/**/__tests__/*", "public/**/__tests__/*"],
   "compilerOptions": {
     "lib": ["ES2021", "DOM", "DOM.Iterable"],
     "resolvePackageJsonExports": false,
@@ -18,7 +18,6 @@
     "baseUrl": ".",
     "noEmit": true,
     "allowImportingTsExtensions": true,
-    "noEmit": true,
     "noUncheckedIndexedAccess": true,
     "exactOptionalPropertyTypes": true,
     "types": ["vitest/importMeta"],

app/gui2/tsconfig.app.vitest.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.app.json",
+  "exclude": [],
+  "compilerOptions": {
+    "lib": [],
+    "types": ["node", "jsdom", "vitest/importMeta"]
+  }
+}

app/gui2/tsconfig.json

@@ -11,10 +11,13 @@
       "path": "./tsconfig.app.json"
     },
     {
-      "path": "./tsconfig.vitest.json"
+      "path": "./tsconfig.app.vitest.json"
     },
     {
       "path": "./tsconfig.server.json"
+    },
+    {
+      "path": "./tsconfig.server.vitest.json"
     }
   ]
 }

app/gui2/tsconfig.node.json

@@ -9,6 +9,7 @@
     "node.env.d.ts"
   ],
   "compilerOptions": {
+    "baseUrl": ".",
     "module": "ESNext",
     "moduleResolution": "Bundler",
     "outDir": "../../node_modules/.cache/tsc",

app/gui2/tsconfig.server.json

@@ -1,12 +1,13 @@
 {
   "extends": "@tsconfig/node18/tsconfig.json",
   "include": ["ydoc-server/**/*", "shared/**/*"],
+  "exclude": ["ydoc-server/**/__tests__/*", "shared/**/__tests__/*"],
   "compilerOptions": {
     "baseUrl": ".",
     "module": "ESNext",
     "moduleResolution": "Bundler",
-    "outDir": "../../node_modules/.cache/tsc",
     "composite": true,
+    "outDir": "../../node_modules/.cache/tsc",
     "types": ["node", "vitest/importMeta"]
   }
 }

app/gui2/tsconfig.server.vitest.json

@@ -0,0 +1,8 @@
+{
+  "extends": "./tsconfig.server.json",
+  "exclude": [],
+  "compilerOptions": {
+    "lib": [],
+    "types": ["node", "vitest/importMeta"]
+  }
+}

app/gui2/tsconfig.vitest.json

@@ -1,15 +0,0 @@
-{
-  "extends": "./tsconfig.app.json",
-  "exclude": [],
-  "compilerOptions": {
-    "composite": true,
-    "lib": [],
-    "outDir": "../../node_modules/.cache/tsc",
-    "types": ["node", "jsdom", "vitest/importMeta"]
-  },
-  "references": [
-    {
-      "path": "../ide-desktop/lib/dashboard/tsconfig.json"
-    }
-  ]
-}

app/gui2/ydoc-server/__tests__/edits.test.ts

@@ -0,0 +1,175 @@
+import { fc, test } from '@fast-check/vitest'
+import { Position, TextEdit } from 'shared/languageServerTypes'
+import { inspect } from 'util'
+import { describe, expect } from 'vitest'
+import * as Y from 'yjs'
+import { applyDiffAsTextEdits, convertDeltaToTextEdits } from '../edits'
+
+// ======================
+// === Test utilities ===
+// ======================
+
+/** Apply text edits intended for language server to a given starting text. Used for verification
+ * during testing if generated edits were correct. This is intentionally a very simple, not
+ * performant implementation.
+ */
+export function applyTextEdits(content: string, edits: TextEdit[]): string {
+  return edits.reduce((c, edit) => {
+    try {
+      const startOffset = lsPositionToTextOffset(c, edit.range.start)
+      const endOffset = lsPositionToTextOffset(c, edit.range.end)
+      return c.slice(0, startOffset) + edit.text + c.slice(endOffset)
+    } catch (e) {
+      throw new Error(
+        `Failed to apply edit ${inspect(edit)} to content:\n${inspect(c)}\n${String(e)}`,
+      )
+    }
+  }, content)
+}
+
+function lsPositionToTextOffset(content: string, pos: Position): number {
+  const lineData = getNthLineLengthAndOffset(content, pos.line)
+  if (pos.character > lineData.length)
+    throw new Error(
+      `Character ${pos.character} is out of bounds for line ${pos.line}. ` +
+        `Line length ${lineData.length}.`,
+    )
+  return lineData.offset + pos.character
+}
+
+function getNthLineLengthAndOffset(content: string, line: number) {
+  const nthLineRegex = new RegExp(`((?:.*\\n){${line}})(.*)(?:\n|$)`)
+  const match = nthLineRegex.exec(content)
+  if (!match) throw new Error(`Line ${line} not found in content:\n${content}`)
+  return {
+    offset: match[1].length,
+    length: match[2].length,
+  }
+}
+
+// ==================
+// === Test suite ===
+// ==================
+
+describe('applyDiffAsTextEdits', () => {
+  test('no change', () => {
+    const edits = applyDiffAsTextEdits(0, 'abcd', 'abcd')
+    expect(edits).toStrictEqual([])
+  })
+  test('simple add', () => {
+    const before = 'abcd'
+    const after = 'abefcd'
+    const edits = applyDiffAsTextEdits(1, before, after)
+    expect(edits).toStrictEqual([
+      {
+        range: { end: { character: 2, line: 1 }, start: { character: 2, line: 1 } },
+        text: 'ef',
+      },
+    ])
+  })
+
+  test('two adds', () => {
+    const before = 'abcd'
+    const after = 'abefcdxy'
+    const edits = applyDiffAsTextEdits(1, before, after)
+    expect(edits).toStrictEqual([
+      {
+        range: { end: { character: 2, line: 1 }, start: { character: 2, line: 1 } },
+        text: 'ef',
+      },
+      {
+        range: { end: { character: 6, line: 1 }, start: { character: 6, line: 1 } },
+        text: 'xy',
+      },
+    ])
+  })
+
+  test.prop({
+    linesBefore: fc.array(fc.string(), { minLength: 1 }),
+    linesAfter: fc.array(fc.string(), { minLength: 1 }),
+    ctx: fc.context(),
+  })('should correctly create edits from random diffs', ({ linesBefore, linesAfter, ctx }) => {
+    const before = linesBefore.join('\n')
+    const after = linesAfter.join('\n')
+    const edits = applyDiffAsTextEdits(0, before, after)
+    for (const edit of edits) {
+      ctx.log(
+        `${edit.range.start.line}:${edit.range.start.character} - ` +
+          `${edit.range.end.line}:${edit.range.end.character} - '${edit.text}'`,
+      )
+    }
+    const applied = applyTextEdits(before, edits)
+    expect(applied).toBe(after)
+  })
+})
+
+describe('convertDeltaToTextEdits', () => {
+  function genDeltaAndResultFromOps(
+    initial: string,
+    ops: ({ insert: [number, string] } | { delete: [number, number] })[],
+  ): { delta: Y.YTextEvent['delta']; result: string } {
+    const doc = new Y.Doc()
+    const ytext = doc.getText()
+    ytext.insert(0, initial)
+    let delta: Y.YTextEvent['delta'] | null = null
+    ytext.observe((e) => {
+      delta = e.delta
+    })
+
+    doc.transact(() => {
+      for (const op of ops) {
+        if ('insert' in op) {
+          ytext.insert(Math.min(op.insert[0], ytext.length), op.insert[1])
+        } else {
+          const idx = Math.min(op.delete[0], ytext.length)
+          const len = Math.min(op.delete[1], ytext.length - idx)
+          ytext.delete(idx, len)
+        }
+      }
+    })
+
+    return {
+      delta: delta ?? [],
+      result: ytext.toString(),
+    }
+  }
+
+  function deltaOp() {
+    return fc.oneof(
+      fc.record({ insert: fc.tuple(fc.nat(), fc.string()) }),
+      fc.record({ delete: fc.tuple(fc.nat(), fc.nat()) }),
+    )
+  }
+
+  test.prop({
+    initialLines: fc.array(fc.string()),
+    ops: fc.array(deltaOp()),
+    deltaCtx: fc.context(),
+    ctx: fc.context(),
+  })(
+    'should correctly translate deltas to random diffs',
+    ({ initialLines, ops, deltaCtx, ctx }) => {
+      const initial = initialLines.join('\n')
+      const { delta, result } = genDeltaAndResultFromOps(initial, ops)
+      for (const op of delta) {
+        if ('retain' in op) {
+          deltaCtx.log(`retain ${op.retain}`)
+        } else if ('delete' in op) {
+          deltaCtx.log(`delete ${op.delete}`)
+        } else {
+          deltaCtx.log(`insert ${op.insert}`)
+        }
+      }
+      const { code, edits } = convertDeltaToTextEdits(initial, delta)
+      for (const edit of edits) {
+        ctx.log(
+          `${edit.range.start.line}:${edit.range.start.character} - ` +
+            `${edit.range.end.line}:${edit.range.end.character} - '${edit.text}'`,
+        )
+      }
+      const applied = applyTextEdits(initial, edits)
+      expect(applied).toBe(result)
+      expect(code).toBe(result)
+    },
+  )
+})

app/gui2/ydoc-server/edits.ts

@@ -0,0 +1,296 @@
+/**
+ * @file A module responsible for translating file edits between the Yjs document updates and the
+ * Language server protocol structures.
+ */
+
+import diff from 'fast-diff'
+import * as json from 'lib0/json'
+import * as Y from 'yjs'
+import { TextEdit } from '../shared/languageServerTypes'
+import { ModuleDoc, NodeMetadata, decodeRange } from '../shared/yjsModel'
+import * as fileFormat from './fileFormat'
+
+interface AppliedUpdates {
+  edits: TextEdit[]
+  newContent: string
+  newMetadata: fileFormat.Metadata
+}
+
+const META_TAG = '#### METADATA ####'
+
+export function applyDocumentUpdates(
+  doc: ModuleDoc,
+  syncedMeta: fileFormat.Metadata,
+  syncedContent: string,
+  contentDelta: Y.YTextEvent['delta'] | null,
+  idMapKeys: Y.YMapEvent<Uint8Array>['keys'] | null,
+  metadataKeys: Y.YMapEvent<NodeMetadata>['keys'] | null,
+): AppliedUpdates {
+  const synced = preParseContent(syncedContent)
+  let newContent = ''
+
+  const allEdits: TextEdit[] = []
+  if (contentDelta && contentDelta.length > 0) {
+    const { code, edits } = convertDeltaToTextEdits(synced.code, contentDelta)
+    newContent += code
+    allEdits.push(...edits)
+  } else {
+    newContent += synced.code
+  }
+
+  const metaStartLine = (newContent.match(/\n/g) ?? []).length
+  let metaContent = META_TAG + '\n'
+
+  if (idMapKeys != null || synced.idMapJson == null || (contentDelta && contentDelta.length > 0)) {
+    const idMapJson = json.stringify(idMapToArray(doc.idMap))
+    metaContent += idMapJson + '\n'
+  } else {
+    metaContent += (synced.idMapJson ?? '[]') + '\n'
+  }
+
+  let newMetadata = syncedMeta
+  if (metadataKeys != null) {
+    const nodeMetadata = { ...syncedMeta.ide.node }
+    for (const [key, op] of metadataKeys) {
+      switch (op.action) {
+        case 'delete':
+          delete nodeMetadata[key]
+          break
+        case 'add':
+        case 'update': {
+          const updatedMeta = doc.metadata.get(key)
+          const oldMeta = nodeMetadata[key] ?? {}
+          if (updatedMeta == null) continue
+          nodeMetadata[key] = {
+            ...oldMeta,
+            position: {
+              vector: [updatedMeta.x, updatedMeta.y],
+            },
+          }
+          break
+        }
+      }
+    }
+    // Update the metadata object without changing the original order of keys.
+    newMetadata = { ...syncedMeta }
+    newMetadata.ide = { ...syncedMeta.ide }
+    newMetadata.ide.node = nodeMetadata
+    const metadataJson = json.stringify(newMetadata)
+    metaContent += metadataJson
+  } else {
+    metaContent += synced.metadataJson ?? '{}'
+  }
+
+  const oldMetaContent = syncedContent.slice(synced.code.length)
+  allEdits.push(...applyDiffAsTextEdits(metaStartLine, oldMetaContent, metaContent))
+  newContent += metaContent
+
+  return {
+    edits: allEdits,
+    newContent,
+    newMetadata: newMetadata,
+  }
+}
+
+export function convertDeltaToTextEdits(
+  prevText: string,
+  contentDelta: Y.YTextEvent['delta'],
+): { code: string; edits: TextEdit[] } {
+  const edits = []
+  let index = 0
+  let newIndex = 0
+  let lineNum = 0
+  let lineStartIdx = 0
+  let code = ''
+  for (const op of contentDelta) {
+    if (op.insert != null && typeof op.insert === 'string') {
+      const pos = {
+        character: newIndex - lineStartIdx,
+        line: lineNum,
+      }
+      // if the last edit was a delete on the same position, we can merge the insert into it
+      const lastEdit = edits[edits.length - 1]
+      if (
+        lastEdit &&
+        lastEdit.text.length === 0 &&
+        lastEdit.range.start.line === pos.line &&
+        lastEdit.range.start.character === pos.character
+      ) {
+        lastEdit.text = op.insert
+      } else {
+        edits.push({ range: { start: pos, end: pos }, text: op.insert })
+      }
+      const numLineBreaks = (op.insert.match(/\n/g) ?? []).length
+      if (numLineBreaks > 0) {
+        lineNum += numLineBreaks
+        lineStartIdx = newIndex + op.insert.lastIndexOf('\n') + 1
+      }
+      code += op.insert
+      newIndex += op.insert.length
+    } else if (op.delete != null) {
+      const start = {
+        character: newIndex - lineStartIdx,
+        line: lineNum,
+      }
+      const deleted = prevText.slice(index, index + op.delete)
+      const numLineBreaks = (deleted.match(/\n/g) ?? []).length
+      const character =
+        numLineBreaks > 0
+          ? deleted.length - (deleted.lastIndexOf('\n') + 1)
+          : newIndex - lineStartIdx + op.delete
+      const end = {
+        character,
+        line: lineNum + numLineBreaks,
+      }
+      edits.push({ range: { start, end }, text: '' })
+      index += op.delete
+    } else if (op.retain != null) {
+      const retained = prevText.slice(index, index + op.retain)
+      const numLineBreaks = (retained.match(/\n/g) ?? []).length
+      lineNum += numLineBreaks
+      if (numLineBreaks > 0) {
+        lineStartIdx = newIndex + retained.lastIndexOf('\n') + 1
+      }
+      code += retained
+      index += op.retain
+      newIndex += op.retain
+    }
+  }
+  code += prevText.slice(index)
+  return { code, edits }
+}
+
+interface PreParsedContent {
+  code: string
+  idMapJson: string | null
+  metadataJson: string | null
+}
+
+export function preParseContent(content: string): PreParsedContent {
+  const splitPoint = content.lastIndexOf(META_TAG)
+  if (splitPoint < 0) {
+    return {
+      code: content,
+      idMapJson: null,
+      metadataJson: null,
+    }
+  }
+  const code = content.slice(0, splitPoint)
+  const metadataString = content.slice(splitPoint + META_TAG.length)
+  const metaLines = metadataString.trim().split('\n')
+  const idMapJson = metaLines[0] ?? null
+  const metadataJson = metaLines[1] ?? null
+  return { code, idMapJson, metadataJson }
+}
+
+function idMapToArray(map: Y.Map<Uint8Array>): fileFormat.IdMapEntry[] {
+  const entries: fileFormat.IdMapEntry[] = []
+  const doc = map.doc!
+  map.forEach((rangeBuffer, id) => {
+    const decoded = decodeRange(rangeBuffer)
+    const index = Y.createAbsolutePositionFromRelativePosition(decoded[0], doc)?.index
+    const endIndex = Y.createAbsolutePositionFromRelativePosition(decoded[1], doc)?.index
+    if (index == null || endIndex == null) return
+    const size = endIndex - index
+    entries.push([{ index: { value: index }, size: { value: size } }, id])
+  })
+  entries.sort(idMapCmp)
+  return entries
+}
+
+function idMapCmp(a: fileFormat.IdMapEntry, b: fileFormat.IdMapEntry) {
+  const val1 = a[0]?.index?.value ?? 0
+  const val2 = b[0]?.index?.value ?? 0
+  if (val1 === val2) {
+    const size1 = a[0]?.size.value ?? 0
+    const size2 = b[0]?.size.value ?? 0
+    return size1 - size2
+  }
+  return val1 - val2
+}
+
+export function applyDiffAsTextEdits(
+  lineOffset: number,
+  oldString: string,
+  newString: string,
+): TextEdit[] {
+  const changes = diff(oldString, newString)
+  let newIndex = 0
+  let lineNum = lineOffset
+  let lineStartIdx = 0
+  const edits = []
+  for (const [op, text] of changes) {
+    if (op === 1) {
+      const pos = {
+        character: newIndex - lineStartIdx,
+        line: lineNum,
+      }
+      edits.push({ range: { start: pos, end: pos }, text })
+      const numLineBreaks = (text.match(/\n/g) ?? []).length
+      if (numLineBreaks > 0) {
+        lineNum += numLineBreaks
+        lineStartIdx = newIndex + text.lastIndexOf('\n') + 1
+      }
+      newIndex += text.length
+    } else if (op === -1) {
+      const start = {
+        character: newIndex - lineStartIdx,
+        line: lineNum,
+      }
+      const numLineBreaks = (text.match(/\n/g) ?? []).length
+      const character =
+        numLineBreaks > 0
+          ? text.length - (text.lastIndexOf('\n') + 1)
+          : newIndex - lineStartIdx + text.length
+      const end = {
+        character,
+        line: lineNum + numLineBreaks,
+      }
+      edits.push({ range: { start, end }, text: '' })
+    } else if (op === 0) {
+      const numLineBreaks = (text.match(/\n/g) ?? []).length
+      lineNum += numLineBreaks
+      if (numLineBreaks > 0) {
+        lineStartIdx = newIndex + text.lastIndexOf('\n') + 1
+      }
+      newIndex += text.length
+    }
+  }
+  return edits
+}
+
+export function prettyPrintDiff(from: string, to: string): string {
+  const colReset = '\x1b[0m'
+  const colRed = '\x1b[31m'
+  const colGreen = '\x1b[32m'
+
+  const diffs = diff(from, to)
+  if (diffs.length === 1 && diffs[0]![0] === 0) return 'No changes'
+  let content = ''
+  for (let i = 0; i < diffs.length; i++) {
+    const [op, text] = diffs[i]!
+    if (op === 1) {
+      content += colGreen + text
+    } else if (op === -1) {
+      content += colRed + text
+    } else if (op === 0) {
+      content += colReset
+      const numNewlines = (text.match(/\n/g) ?? []).length
+      if (numNewlines < 2) {
+        content += text
+      } else {
+        const firstNewline = text.indexOf('\n')
+        const lastNewline = text.lastIndexOf('\n')
+        const firstLine = text.slice(0, firstNewline + 1)
+        const lastLine = text.slice(lastNewline + 1)
+        const isFirst = i === 0
+        const isLast = i === diffs.length - 1
+        if (!isFirst) content += firstLine
+        if (!isFirst && !isLast) content += '...\n'
+        if (!isLast) content += lastLine
+      }
+    }
+  }
+  content += colReset
+  return content
+}

app/gui2/ydoc-server/languageServerSession.ts

@@ -1,13 +1,11 @@
 import { Client, RequestManager, WebSocketTransport } from '@open-rpc/client-js'
-import diff from 'fast-diff'
 import { simpleDiffString } from 'lib0/diff'
-import * as json from 'lib0/json'
 import * as map from 'lib0/map'
 import { ObservableV2 } from 'lib0/observable'
 import * as random from 'lib0/random'
 import * as Y from 'yjs'
 import { LanguageServer, computeTextChecksum } from '../shared/languageServer'
-import { Checksum, Path, TextEdit } from '../shared/languageServerTypes'
+import { Checksum, Path } from '../shared/languageServerTypes'
 import {
   DistributedProject,
   ExprId,
@@ -15,8 +13,8 @@ import {
   ModuleDoc,
   NodeMetadata,
   Uuid,
-  decodeRange,
 } from '../shared/yjsModel'
+import { applyDocumentUpdates, preParseContent, prettyPrintDiff } from './edits'
 import * as fileFormat from './fileFormat'
 import { WSSharedDoc } from './ydoc'
 
@@ -192,10 +190,6 @@ const pushPathSegment = (path: Path, segment: string): Path => {
   }
 }
 
-const META_TAG = '#### METADATA ####'
-
-type IdMapEntry = [{ index: { value: number }; size: { value: number } }, string]
-
 enum LsSyncState {
   Closed,
   Opening,
@@ -336,102 +330,52 @@ class ModulePersistence extends ObservableV2<{ removed: () => void }> {
     if (this.syncedContent == null || this.syncedVersion == null) return
     if (contentDelta == null && idMapKeys == null && metadataKeys == null) return
 
-    const synced = preParseContent(this.syncedContent)
-
-    let newContent = ''
-
-    const allEdits: TextEdit[] = []
-    if (contentDelta && contentDelta.length > 0) {
-      const { code, edits } = convertDeltaToTextEdits(synced.code, contentDelta)
-      newContent += code
-      allEdits.push(...edits)
-    } else {
-      newContent += synced.code
-    }
-
-    const metaStartLine = (newContent.match(/\n/g) ?? []).length
-    let metaContent = META_TAG + '\n'
-
-    if (
-      idMapKeys != null ||
-      synced.idMapJson == null ||
-      (contentDelta && contentDelta.length > 0)
-    ) {
-      const idMapJson = json.stringify(idMapToArray(this.doc.idMap))
-      metaContent += idMapJson + '\n'
-    } else {
-      metaContent += (synced.idMapJson ?? '[]') + '\n'
-    }
-
-    const nodeMetadata = this.syncedMeta.ide.node
-    if (metadataKeys != null) {
-      for (const [key, op] of metadataKeys) {
-        switch (op.action) {
-          case 'delete':
-            delete nodeMetadata[key]
-            break
-          case 'add':
-          case 'update': {
-            const updatedMeta = this.doc.metadata.get(key)
-            const oldMeta = nodeMetadata[key] ?? {}
-            if (updatedMeta == null) continue
-            nodeMetadata[key] = {
-              ...oldMeta,
-              position: {
-                vector: [updatedMeta.x, updatedMeta.y],
-              },
-            }
-            break
-          }
-        }
-      }
-
-      const metadataJson = json.stringify(this.syncedMeta)
-      metaContent += metadataJson
-    } else {
-      metaContent += synced.metadataJson ?? '{}'
-    }
-
-    const oldMetaContent = this.syncedContent.slice(synced.code.length)
-    allEdits.push(...applyDiffAsTextEdits(metaStartLine, oldMetaContent, metaContent))
-    newContent += metaContent
+    const { edits, newContent, newMetadata } = applyDocumentUpdates(
+      this.doc,
+      this.syncedMeta,
+      this.syncedContent,
+      contentDelta,
+      idMapKeys,
+      metadataKeys,
+    )
 
     const newVersion = computeTextChecksum(newContent)
 
     if (DEBUG_LOG_SYNC) {
       console.log(' === changes === ')
-      console.log('number of edits:', allEdits.length)
+      console.log('number of edits:', edits.length)
       console.log('metadata:', metadataKeys)
       console.log('content:', contentDelta)
       console.log('idMap:', idMapKeys)
-      if (allEdits.length > 0) {
+      if (edits.length > 0) {
         console.log('version:', this.syncedVersion, '->', newVersion)
         console.log('Content diff:')
         console.log(prettyPrintDiff(this.syncedContent, newContent))
       }
       console.log(' =============== ')
     }
-    if (allEdits.length === 0) {
+    if (edits.length === 0) {
+      if (newVersion !== this.syncedVersion) {
+        console.error('Version mismatch:', this.syncedVersion, '->', newVersion)
+      }
       return
     }
 
-    const execute = (contentDelta && contentDelta.length > 0) || metadataKeys != null
-
     this.changeState(LsSyncState.WritingFile)
     const apply = this.ls.applyEdit(
       {
         path: this.path,
-        edits: allEdits,
+        edits,
         oldVersion: this.syncedVersion,
         newVersion,
       },
-      execute,
+      true,
     )
     return (this.lastAction = apply.then(
       () => {
         this.syncedContent = newContent
         this.syncedVersion = newVersion
-        this.syncedMeta.ide.node = nodeMetadata
+        this.syncedMeta = newMetadata
         this.changeState(LsSyncState.Synchronized)
       },
       (e) => {
@@ -582,237 +526,3 @@ class ModulePersistence extends ObservableV2<{ removed: () => void }> {
     return Promise.resolve()
   }
 }
-
-function applyDiffAsTextEdits(
-  lineOffset: number,
-  oldString: string,
-  newString: string,
-): TextEdit[] {
-  const changes = diff(oldString, newString)
-  let newIndex = 0
-  let lineNum = lineOffset
-  let lineStartIdx = 0
-  const edits = []
-  for (const [op, text] of changes) {
-    if (op === 1) {
-      const pos = {
-        character: newIndex - lineStartIdx,
-        line: lineNum,
-      }
-      edits.push({ range: { start: pos, end: pos }, text })
-      const numLineBreaks = (text.match(/\n/g) ?? []).length
-      if (numLineBreaks > 0) {
-        lineStartIdx = newIndex + text.lastIndexOf('\n') + 1
-      }
-      newIndex += text.length
-    } else if (op === -1) {
-      const start = {
-        character: newIndex - lineStartIdx,
-        line: lineNum,
-      }
-      const numLineBreaks = (text.match(/\n/g) ?? []).length
-      const character =
-        numLineBreaks > 0 ? text.lastIndexOf('\n') + 1 : newIndex - lineStartIdx + text.length
-      const end = {
-        character,
-        line: lineNum + numLineBreaks,
-      }
-      edits.push({ range: { start, end }, text: '' })
-    } else if (op === 0) {
-      const numLineBreaks = (text.match(/\n/g) ?? []).length
-      lineNum += numLineBreaks
-      if (numLineBreaks > 0) {
-        lineStartIdx = newIndex + text.lastIndexOf('\n') + 1
-      }
-      newIndex += text.length
-    }
-  }
-  return edits
-}
-
-function convertDeltaToTextEdits(
-  prevText: string,
-  contentDelta: Y.YTextEvent['delta'],
-): { code: string; edits: TextEdit[] } {
-  const edits = []
-  let index = 0
-  let newIndex = 0
-  let lineNum = 0
-  let lineStartIdx = 0
-  let code = ''
-  for (const op of contentDelta) {
-    if (op.insert != null && typeof op.insert === 'string') {
-      const pos = {
-        character: newIndex - lineStartIdx,
-        line: lineNum,
-      }
-      // if the last edit was a delete on the same position, we can merge the insert into it
-      const lastEdit = edits[edits.length - 1]
-      if (
-        lastEdit &&
-        lastEdit.text.length === 0 &&
-        lastEdit.range.start.line === pos.line &&
-        lastEdit.range.start.character === pos.character
-      ) {
-        lastEdit.text = op.insert
-      } else {
-        edits.push({ range: { start: pos, end: pos }, text: op.insert })
-      }
-      const numLineBreaks = (op.insert.match(/\n/g) ?? []).length
-      if (numLineBreaks > 0) {
-        lineStartIdx = newIndex + op.insert.lastIndexOf('\n') + 1
-      }
-      code += op.insert
-      newIndex += op.insert.length
-    } else if (op.delete != null) {
-      const start = {
-        character: newIndex - lineStartIdx,
-        line: lineNum,
-      }
-      const deleted = prevText.slice(index, index + op.delete)
-      const numLineBreaks = (deleted.match(/\n/g) ?? []).length
-      const character =
-        numLineBreaks > 0 ? deleted.lastIndexOf('\n') + 1 : newIndex - lineStartIdx + op.delete
-      const end = {
-        character,
-        line: lineNum + numLineBreaks,
-      }
-      edits.push({ range: { start, end }, text: '' })
-      index += op.delete
-    } else if (op.retain != null) {
-      const retained = prevText.slice(index, index + op.retain)
-      const numLineBreaks = (retained.match(/\n/g) ?? []).length
-      lineNum += numLineBreaks
-      if (numLineBreaks > 0) {
-        lineStartIdx = newIndex + retained.lastIndexOf('\n') + 1
-      }
-      code += retained
-      index += op.retain
-      newIndex += op.retain
-    }
-  }
-  code += prevText.slice(index)
-  return { code, edits }
-}
-
-interface PreParsedContent {
-  code: string
-  idMapJson: string | null
-  metadataJson: string | null
-}
-
-function preParseContent(content: string): PreParsedContent {
-  const splitPoint = content.lastIndexOf(META_TAG)
-  if (splitPoint < 0) {
-    return {
-      code: content,
-      idMapJson: null,
-      metadataJson: null,
-    }
-  }
-  const code = content.slice(0, splitPoint)
-  const metadataString = content.slice(splitPoint + META_TAG.length)
-  const metaLines = metadataString.trim().split('\n')
-  const idMapJson = metaLines[0] ?? null
-  const metadataJson = metaLines[1] ?? null
-  return { code, idMapJson, metadataJson }
-}
-
-function idMapToArray(map: Y.Map<Uint8Array>): IdMapEntry[] {
-  const entries: IdMapEntry[] = []
-  const doc = map.doc!
-  map.forEach((rangeBuffer, id) => {
-    const decoded = decodeRange(rangeBuffer)
-    const index = Y.createAbsolutePositionFromRelativePosition(decoded[0], doc)?.index
-    const endIndex = Y.createAbsolutePositionFromRelativePosition(decoded[1], doc)?.index
-    if (index == null || endIndex == null) return
-    const size = endIndex - index
-    entries.push([{ index: { value: index }, size: { value: size } }, id])
-  })
-  entries.sort(idMapCmp)
-  return entries
-}
-
-function idMapCmp(a: IdMapEntry, b: IdMapEntry) {
-  const val1 = a[0]?.index?.value ?? 0
-  const val2 = b[0]?.index?.value ?? 0
-  if (val1 === val2) {
-    const size1 = a[0]?.size.value ?? 0
-    const size2 = b[0]?.size.value ?? 0
-    return size1 - size2
-  }
-  return val1 - val2
-}
-
-if (import.meta.vitest) {
-  const { test, expect, describe } = import.meta.vitest
-
-  describe('applyDiffAsTextEdits', () => {
-    test('no change', () => {
-      const edits = applyDiffAsTextEdits(0, 'abcd', 'abcd')
-      expect(edits).toStrictEqual([])
-    })
-    test('simple add', () => {
-      const before = 'abcd'
-      const after = 'abefcd'
-      const edits = applyDiffAsTextEdits(1, before, after)
-      expect(edits).toStrictEqual([
-        {
-          range: { end: { character: 2, line: 1 }, start: { character: 2, line: 1 } },
-          text: 'ef',
-        },
-      ])
-    })
-    test('two adds', () => {
-      const before = 'abcd'
-      const after = 'abefcdxy'
-      const edits = applyDiffAsTextEdits(1, before, after)
-      expect(edits).toStrictEqual([
-        {
-          range: { end: { character: 2, line: 1 }, start: { character: 2, line: 1 } },
-          text: 'ef',
-        },
-        {
-          range: { end: { character: 6, line: 1 }, start: { character: 6, line: 1 } },
-          text: 'xy',
-        },
-      ])
-    })
-  })
-}
-
-function prettyPrintDiff(from: string, to: string): string {
-  const colReset = '\x1b[0m'
-  const colRed = '\x1b[31m'
-  const colGreen = '\x1b[32m'
-
-  const diffs = diff(from, to)
-  if (diffs.length === 1 && diffs[0]![0] === 0) return 'No changes'
-  let content = ''
-  for (let i = 0; i < diffs.length; i++) {
-    const [op, text] = diffs[i]!
-    if (op === 1) {
-      content += colGreen + text
-    } else if (op === -1) {
-      content += colRed + text
-    } else if (op === 0) {
-      content += colReset
-      const numNewlines = (text.match(/\n/g) ?? []).length
-      if (numNewlines < 2) {
-        content += text
-      } else {
-        const firstNewline = text.indexOf('\n')
-        const lastNewline = text.lastIndexOf('\n')
-        const firstLine = text.slice(0, firstNewline + 1)
-        const lastLine = text.slice(lastNewline + 1)
-        const isFirst = i === 0
-        const isLast = i === diffs.length - 1
-        if (!isFirst) content += firstLine
-        if (!isFirst && !isLast) content += '...\n'
-        if (!isLast) content += lastLine
-      }
-    }
-  }
-  content += colReset
-  return content
-}

app/ide-desktop/lib/content/package.json

@@ -16,7 +16,6 @@
   },
   "scripts": {
     "typecheck": "tsc --noEmit",
-    "lint": "eslint src",
     "build": "tsx bundle.ts",
     "watch": "tsx watch.ts",
     "start": "tsx start.ts"

app/ide-desktop/lib/dashboard/package.json

@@ -5,7 +5,6 @@
   "private": true,
   "scripts": {
     "typecheck": "tsc",
-    "lint": "eslint src",
     "build": "tsx bundle.ts",
     "watch": "tsx watch.ts",
     "start": "tsx start.ts",

app/ide-desktop/lib/types/tsconfig.json

@@ -1,8 +1,10 @@
 {
+  "include": ["./*"],
   "compilerOptions": {
     "types": ["node"],
     "lib": ["ES2015", "DOM"],
-    "skipLibCheck": false
+    "skipLibCheck": false,
+    "outDir": "../../../../node_modules/.cache/tsc"
   },
   "extends": "../../tsconfig.json"
 }

app/ide-desktop/package.json

@@ -30,7 +30,8 @@
     "build-dashboard": "npm run build --workspace enso-dashboard",
     "test-dashboard": "npm run test --workspace enso-dashboard",
     "typecheck": "npx tsc -p lib/types/tsconfig.json",
-    "lint": "eslint ."
+    "lint-only": "eslint .",
+    "lint": "npm run --workspace=enso-gui2 compile-server && npm run lint-only"
   },
   "dependencies": {
     "esbuild-plugin-inline-image": "^0.0.9",

package-lock.json

@@ -29,6 +29,7 @@
       "hasInstallScript": true,
       "dependencies": {
         "@babel/parser": "^7.22.16",
+        "@fast-check/vitest": "^0.0.8",
         "@open-rpc/client-js": "^1.8.1",
         "@vueuse/core": "^10.4.1",
         "codemirror": "^6.0.1",
@@ -2572,6 +2573,27 @@
         "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
       }
     },
+    "node_modules/@fast-check/vitest": {
+      "version": "0.0.8",
+      "resolved": "https://registry.npmjs.org/@fast-check/vitest/-/vitest-0.0.8.tgz",
+      "integrity": "sha512-cFrcu7nwH+rk1qm1J4YrM1k4MIwvIHG7MrQUMGizqPe58XsvvpZz0X9Xkx1e+xaNg9s1YRVTd241WSR0dK/SpQ==",
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/dubzzz"
+        },
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/fast-check"
+        }
+      ],
+      "dependencies": {
+        "fast-check": "^3.0.0"
+      },
+      "peerDependencies": {
+        "vitest": ">=0.28.1"
+      }
+    },
     "node_modules/@fortawesome/fontawesome-common-types": {
       "version": "6.4.2",
       "hasInstallScript": true,
@@ -8317,6 +8339,27 @@
       "version": "1.0.0",
       "license": "MIT"
     },
+    "node_modules/fast-check": {
+      "version": "3.13.1",
+      "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-3.13.1.tgz",
+      "integrity": "sha512-Xp00tFuWd83i8rbG/4wU54qU+yINjQha7bXH2N4ARNTkyOimzHtUBJ5+htpdXk7RMaCOD/j2jxSjEt9u9ZPNeQ==",
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/dubzzz"
+        },
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/fast-check"
+        }
+      ],
+      "dependencies": {
+        "pure-rand": "^6.0.0"
+      },
+      "engines": {
+        "node": ">=8.0.0"
+      }
+    },
     "node_modules/fast-deep-equal": {
       "version": "3.1.3",
       "license": "MIT"
@@ -12560,6 +12603,21 @@
         "node": ">=8"
       }
     },
+    "node_modules/pure-rand": {
+      "version": "6.0.4",
+      "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.0.4.tgz",
+      "integrity": "sha512-LA0Y9kxMYv47GIPJy6MI84fqTd2HmYZI83W/kM/SkKfDlajnZYfmXFTxkbY+xSBPkLJxltMa9hIkmdc29eguMA==",
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/dubzzz"
+        },
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/fast-check"
+        }
+      ]
+    },
     "node_modules/qs": {
       "version": "6.5.3",
       "license": "BSD-3-Clause",