enso-org/enso
1
1
Fork 0
mirror of https://github.com/enso-org/enso.git synced 2024-11-22 03:32:23 +03:00
Code Issues Projects Releases Wiki Activity

Render tables in documentation. (#11564)

Browse Source 
* Render tables in documentation.

Also:
- Separate parser for our flavor of Markdown from the CodeMirror integration;
  move the parser into ydoc-shared and use for Markdown line-wrapping.
- Introduce our own version of yCollab extension; initially just the upstream
  version translated to Typescript and our code style.
- Refactor CodeEditor.

* CHANGELOG, prettier

* Apply @farmaazon review.

* Fix

* Lint

* Cleanup

* Integration tests for GraphNodeComment

Also a little refactoring in preparation for new implementation.

* Workaround stuck CI

* Revert "Workaround stuck CI"

This reverts commit 74313842ba.

* Fix merge

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Kaz Wesley 2024-11-20 02:40:24 -08:00 committed by somebody1234
parent 1477051a69
commit 3a33e81990
39 changed files with 1991 additions and 808 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -28,6 +28,7 @@
clipboard or by drag'n'dropping image files.
- ["Write" button in component menu allows to evaluate it separately from the
rest of the workflow][11523].
- [The documentation editor can now display tables][11564]
[11151]: https://github.com/enso-org/enso/pull/11151
[11271]: https://github.com/enso-org/enso/pull/11271
@ -46,6 +47,7 @@
[11469]: https://github.com/enso-org/enso/pull/11469
[11547]: https://github.com/enso-org/enso/pull/11547
[11523]: https://github.com/enso-org/enso/pull/11523
[11564]: https://github.com/enso-org/enso/pull/11564
#### Enso Standard Library

7
LICENSE
View File

@ -199,3 +199,10 @@
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
---
This project includes components that are licensed under the MIT license. The
full text of the MIT license and its copyright notice can be found in the
`app/licenses/` directory.

1
app/gui/e2e/project-view/locate.ts
View File

@ -85,6 +85,7 @@ export const componentBrowser = componentLocator('.ComponentBrowser')
export const nodeOutputPort = componentLocator('.outputPortHoverArea')
export const smallPlusButton = componentLocator('.SmallPlusButton')
export const editorRoot = componentLocator('.EditorRoot')
export const nodeComment = componentLocator('.GraphNodeComment div[contentEditable]')
/**
* A not-selected variant of Component Browser Entry.

12
app/gui/e2e/project-view/nodeClipboard.spec.ts
View File

@ -33,8 +33,8 @@ test('Copy node with comment', async ({ page }) => {
// Check state before operation.
const originalNodes = await locate.graphNode(page).count()
await expect(page.locator('.GraphNodeComment')).toExist()
const originalNodeComments = await page.locator('.GraphNodeComment').count()
await expect(locate.nodeComment(page)).toExist()
const originalNodeComments = await locate.nodeComment(page).count()
// Select a node.
const nodeToCopy = locate.graphNodeByBinding(page, 'final')
@ -48,7 +48,7 @@ test('Copy node with comment', async ({ page }) => {
// Node and comment have been copied.
await expect(locate.graphNode(page)).toHaveCount(originalNodes + 1)
await expect(page.locator('.GraphNodeComment')).toHaveCount(originalNodeComments + 1)
await expect(locate.nodeComment(page)).toHaveCount(originalNodeComments + 1)
})
test('Copy multiple nodes', async ({ page }) => {
@ -56,8 +56,8 @@ test('Copy multiple nodes', async ({ page }) => {
// Check state before operation.
const originalNodes = await locate.graphNode(page).count()
await expect(page.locator('.GraphNodeComment')).toExist()
const originalNodeComments = await page.locator('.GraphNodeComment').count()
await expect(locate.nodeComment(page)).toExist()
const originalNodeComments = await locate.nodeComment(page).count()
// Select some nodes.
const node1 = locate.graphNodeByBinding(page, 'final')
@ -76,7 +76,7 @@ test('Copy multiple nodes', async ({ page }) => {
// Nodes and comment have been copied.
await expect(locate.graphNode(page)).toHaveCount(originalNodes + 2)
// `final` node has a comment.
await expect(page.locator('.GraphNodeComment')).toHaveCount(originalNodeComments + 1)
await expect(locate.nodeComment(page)).toHaveCount(originalNodeComments + 1)
// Check that two copied nodes are isolated, i.e. connected to each other, not original nodes.
await expect(locate.graphNodeByBinding(page, 'prod1')).toBeVisible()
await expect(locate.graphNodeByBinding(page, 'final1')).toBeVisible()

75
app/gui/e2e/project-view/nodeComments.spec.ts Normal file
View File

@ -0,0 +1,75 @@
import test from 'playwright/test'
import * as actions from './actions'
import { expect } from './customExpect'
import { CONTROL_KEY } from './keyboard'
import * as locate from './locate'
test('Edit comment by click', async ({ page }) => {
await actions.goToGraph(page)
const nodeComment = locate.nodeComment(locate.graphNodeByBinding(page, 'final'))
await expect(nodeComment).toHaveText('This node can be entered')
await nodeComment.click()
await page.keyboard.press(`${CONTROL_KEY}+A`)
const NEW_COMMENT = 'New comment text'
await nodeComment.fill(NEW_COMMENT)
await page.keyboard.press(`Enter`)
await expect(nodeComment).not.toBeFocused()
await expect(nodeComment).toHaveText(NEW_COMMENT)
})
test('Start editing comment via menu', async ({ page }) => {
await actions.goToGraph(page)
const node = locate.graphNodeByBinding(page, 'final')
await node.click()
await locate.circularMenu(node).getByRole('button', { name: 'More' }).click()
await locate.circularMenu(node).getByRole('button', { name: 'Comment' }).click()
await expect(locate.nodeComment(node)).toBeFocused()
})
test('Add new comment via menu', async ({ page }) => {
await actions.goToGraph(page)
const INITIAL_NODE_COMMENTS = 1
await expect(locate.nodeComment(page)).toHaveCount(INITIAL_NODE_COMMENTS)
const node = locate.graphNodeByBinding(page, 'data')
const nodeComment = locate.nodeComment(node)
await node.click()
await locate.circularMenu(node).getByRole('button', { name: 'More' }).click()
await locate.circularMenu(node).getByRole('button', { name: 'Comment' }).click()
await expect(locate.nodeComment(node)).toBeFocused()
const NEW_COMMENT = 'New comment text'
await nodeComment.fill(NEW_COMMENT)
await page.keyboard.press(`Enter`)
await expect(nodeComment).not.toBeFocused()
await expect(nodeComment).toHaveText(NEW_COMMENT)
await expect(locate.nodeComment(page)).toHaveCount(INITIAL_NODE_COMMENTS + 1)
})
test('Delete comment by clearing text', async ({ page }) => {
await actions.goToGraph(page)
const nodeComment = locate.nodeComment(locate.graphNodeByBinding(page, 'final'))
await expect(nodeComment).toHaveText('This node can be entered')
await nodeComment.click()
await page.keyboard.press(`${CONTROL_KEY}+A`)
await page.keyboard.press(`Delete`)
await page.keyboard.press(`Enter`)
await expect(nodeComment).not.toExist()
})
test('URL added to comment is rendered as link', async ({ page }) => {
await actions.goToGraph(page)
const nodeComment = locate.nodeComment(locate.graphNodeByBinding(page, 'final'))
await expect(nodeComment).toHaveText('This node can be entered')
await expect(nodeComment.locator('a')).not.toExist()
await nodeComment.click()
await page.keyboard.press(`${CONTROL_KEY}+A`)
const NEW_COMMENT = "Here's a URL: https://example.com"
await nodeComment.fill(NEW_COMMENT)
await page.keyboard.press(`Enter`)
await expect(nodeComment).not.toBeFocused()
await expect(nodeComment).toHaveText(NEW_COMMENT)
await expect(nodeComment.locator('a')).toHaveCount(1)
})

3
app/gui/e2e/project-view/undoRedo.spec.ts
View File

@ -44,7 +44,8 @@ test('Removing node', async ({ page }) => {
await page.keyboard.press(`${CONTROL_KEY}+Z`)
await expect(locate.graphNode(page)).toHaveCount(nodesCount)
await expect(deletedNode.locator('.WidgetToken')).toHaveText(['Main', '.', 'func1', 'prod'])
await expect(deletedNode.locator('.GraphNodeComment')).toHaveText('This node can be entered')
await expect(locate.nodeComment(deletedNode)).toHaveText('This node can be entered')
const restoredBBox = await deletedNode.boundingBox()
expect(restoredBBox).toEqual(deletedNodeBBox)

4
app/gui/package.json
View File

@ -22,7 +22,7 @@
"build-cloud": "cross-env CLOUD_BUILD=true corepack pnpm run build",
"preview": "vite preview",
"//": "max-warnings set to 41 to match the amount of warnings introduced by the new react compiler. Eventual goal is to remove all the warnings.",
"lint": "eslint . --max-warnings=41",
"lint": "eslint . --max-warnings=39",
"format": "prettier --version && prettier --write src/ && eslint . --fix",
"dev:vite": "vite",
"test": "corepack pnpm run /^^^^test:.*/",
@ -94,7 +94,6 @@
"@lexical/plain-text": "^0.16.0",
"@lexical/utils": "^0.16.0",
"@lezer/common": "^1.1.0",
"@lezer/markdown": "^1.3.1",
"@lezer/highlight": "^1.1.6",
"@noble/hashes": "^1.4.0",
"@vueuse/core": "^10.4.1",
@ -118,7 +117,6 @@
"veaury": "^2.3.18",
"vue": "^3.5.2",
"vue-component-type-helpers": "^2.0.29",
"y-codemirror.next": "^0.3.2",
"y-protocols": "^1.0.5",
"y-textarea": "^1.0.0",
"y-websocket": "^1.5.0",

381
app/gui/src/project-view/components/CodeEditor.vue
View File

@ -1,382 +1,13 @@
<script setup lang="ts">
import type { ChangeSet, Diagnostic, Highlighter } from '@/components/CodeEditor/codemirror'
import EditorRoot from '@/components/EditorRoot.vue'
import { useGraphStore, type NodeId } from '@/stores/graph'
import { useProjectStore } from '@/stores/project'
import { useSuggestionDbStore } from '@/stores/suggestionDatabase'
import { useAutoBlur } from '@/util/autoBlur'
import { unwrap } from '@/util/data/result'
import { qnJoin, tryQualifiedName } from '@/util/qualifiedName'
import { EditorSelection } from '@codemirror/state'
import * as iter from 'enso-common/src/utilities/data/iter'
import { createDebouncer } from 'lib0/eventloop'
import type { ComponentInstance } from 'vue'
import { computed, onMounted, onUnmounted, ref, shallowRef, watch, watchEffect } from 'vue'
import { MutableModule } from 'ydoc-shared/ast'
import { textChangeToEdits, type SourceRangeEdit } from 'ydoc-shared/util/data/text'
import { rangeEncloses, type Origin } from 'ydoc-shared/yjsModel'
import { defineAsyncComponent } from 'vue'
// Use dynamic imports to aid code splitting. The codemirror dependency is quite large.
const {
Annotation,
StateEffect,
StateField,
bracketMatching,
foldGutter,
lintGutter,
highlightSelectionMatches,
minimalSetup,
EditorState,
EditorView,
syntaxHighlighting,
defaultHighlightStyle,
tooltips,
enso,
linter,
forceLinting,
lsDiagnosticsToCMDiagnostics,
hoverTooltip,
textEditToChangeSpec,
} = await import('@/components/CodeEditor/codemirror')
const projectStore = useProjectStore()
const graphStore = useGraphStore()
const suggestionDbStore = useSuggestionDbStore()
const editorRoot = ref<ComponentInstance<typeof EditorRoot>>()
const rootElement = computed(() => editorRoot.value?.rootElement)
useAutoBlur(rootElement)
const executionContextDiagnostics = shallowRef<Diagnostic[]>([])
// Effect that can be applied to the document to invalidate the linter state.
const diagnosticsUpdated = StateEffect.define()
// State value that is perturbed by any `diagnosticsUpdated` effect.
const diagnosticsVersion = StateField.define({
create: (_state) => 0,
update: (value, transaction) => {
for (const effect of transaction.effects) {
if (effect.is(diagnosticsUpdated)) value += 1
}
return value
},
})
const expressionUpdatesDiagnostics = computed(() => {
const updates = projectStore.computedValueRegistry.db
const panics = updates.type.reverseLookup('Panic')
const errors = updates.type.reverseLookup('DataflowError')
const diagnostics: Diagnostic[] = []
for (const externalId of iter.chain(panics, errors)) {
const update = updates.get(externalId)
if (!update) continue
const astId = graphStore.db.idFromExternal(externalId)
if (!astId) continue
const span = graphStore.moduleSource.getSpan(astId)
if (!span) continue
const [from, to] = span
switch (update.payload.type) {
case 'Panic': {
diagnostics.push({ from, to, message: update.payload.message, severity: 'error' })
break
}
case 'DataflowError': {
const error = projectStore.dataflowErrors.lookup(externalId)
if (error?.value?.message) {
diagnostics.push({ from, to, message: error.value.message, severity: 'error' })
}
break
}
}
}
return diagnostics
})
// == CodeMirror editor setup ==
// Disable EditContext API because of https://github.com/codemirror/dev/issues/1458.
;(EditorView as any).EDIT_CONTEXT = false
const editorView = new EditorView()
const viewInitialized = ref(false)
watchEffect(() => {
const module = projectStore.module
if (!module) return
editorView.setState(
EditorState.create({
extensions: [
minimalSetup,
updateListener(),
diagnosticsVersion,
syntaxHighlighting(defaultHighlightStyle as Highlighter),
bracketMatching(),
foldGutter(),
lintGutter(),
highlightSelectionMatches(),
tooltips({ position: 'absolute' }),
hoverTooltip((ast, syn) => {
const dom = document.createElement('div')
const astSpan = ast.span()
let foundNode: NodeId | undefined
for (const [id, node] of graphStore.db.nodeIdToNode.entries()) {
const rootSpan = graphStore.moduleSource.getSpan(node.rootExpr.id)
if (rootSpan && rangeEncloses(rootSpan, astSpan)) {
foundNode = id
break
}
}
const expressionInfo = foundNode && graphStore.db.getExpressionInfo(foundNode)
const nodeColor = foundNode && graphStore.db.getNodeColorStyle(foundNode)
if (foundNode != null) {
dom
.appendChild(document.createElement('div'))
.appendChild(document.createTextNode(`AST ID: ${foundNode}`))
}
if (expressionInfo != null) {
dom
.appendChild(document.createElement('div'))
.appendChild(document.createTextNode(`Type: ${expressionInfo.typename ?? 'Unknown'}`))
}
if (expressionInfo?.profilingInfo[0] != null) {
const profile = expressionInfo.profilingInfo[0]
const executionTime = (profile.ExecutionTime.nanoTime / 1_000_000).toFixed(3)
const text = `Execution Time: ${executionTime}ms`
dom
.appendChild(document.createElement('div'))
.appendChild(document.createTextNode(text))
}
dom
.appendChild(document.createElement('div'))
.appendChild(document.createTextNode(`Syntax: ${syn.toString()}`))
const method = expressionInfo?.methodCall?.methodPointer
if (method != null) {
const moduleName = tryQualifiedName(method.module)
const methodName = tryQualifiedName(method.name)
const qualifiedName = qnJoin(unwrap(moduleName), unwrap(methodName))
const [id] = suggestionDbStore.entries.nameToId.lookup(qualifiedName)
const suggestionEntry = id != null ? suggestionDbStore.entries.get(id) : undefined
if (suggestionEntry != null) {
const groupNode = dom.appendChild(document.createElement('div'))
groupNode.appendChild(document.createTextNode('Group: '))
const groupNameNode = groupNode.appendChild(document.createElement('span'))
groupNameNode.appendChild(document.createTextNode(`${method.module}.${method.name}`))
if (nodeColor) {
groupNameNode.style.color = nodeColor
}
}
}
return { dom }
}),
enso(),
linter(
() => [...executionContextDiagnostics.value, ...expressionUpdatesDiagnostics.value],
{
needsRefresh(update) {
return (
update.state.field(diagnosticsVersion) !==
update.startState.field(diagnosticsVersion)
)
},
},
),
],
}),
)
viewInitialized.value = true
})
function changeSetToTextEdits(changes: ChangeSet) {
const textEdits = new Array<SourceRangeEdit>()
changes.iterChanges((from, to, _fromB, _toB, insert) =>
textEdits.push({ range: [from, to], insert: insert.toString() }),
)
return textEdits
}
let pendingChanges: ChangeSet | undefined
let currentModule: MutableModule | undefined
/** Set the editor contents the current module state, discarding any pending editor-initiated changes. */
function resetView() {
console.info(`Resetting the editor to the module code.`)
pendingChanges = undefined
currentModule = undefined
const viewText = editorView.state.doc.toString()
const code = graphStore.moduleSource.text
editorView.dispatch({
changes: textChangeToEdits(viewText, code).map(textEditToChangeSpec),
annotations: synchronizedModule.of(graphStore.startEdit()),
})
}
/** Apply any pending changes to the currently-synchronized module, clearing the set of pending changes. */
function commitPendingChanges() {
if (!pendingChanges || !currentModule) return
try {
currentModule.applyTextEdits(changeSetToTextEdits(pendingChanges), graphStore.viewModule)
graphStore.commitEdit(currentModule, undefined, 'local:userAction:CodeEditor')
} catch (error) {
console.error(`Code Editor failed to modify module`, error)
resetView()
}
pendingChanges = undefined
}
function updateListener() {
const debouncer = createDebouncer(0)
return EditorView.updateListener.of((update) => {
for (const transaction of update.transactions) {
const newModule = transaction.annotation(synchronizedModule)
if (newModule) {
// Flush the pipeline of edits that were based on the old module.
commitPendingChanges()
currentModule = newModule
} else if (transaction.docChanged && currentModule) {
pendingChanges =
pendingChanges ? pendingChanges.compose(transaction.changes) : transaction.changes
// Defer the update until after pending events have been processed, so that if changes are arriving faster than
// we would be able to apply them individually we coalesce them to keep up.
debouncer(commitPendingChanges)
}
}
})
}
let needResync = false
// Indicates a change updating the text to correspond to the given module state.
const synchronizedModule = Annotation.define<MutableModule>()
watch(
viewInitialized,
(ready) => {
if (ready) graphStore.moduleSource.observe(observeSourceChange)
},
{ immediate: true },
const LazyCodeEditor = defineAsyncComponent(
() => import('@/components/CodeEditor/CodeEditorImpl.vue'),
)
onUnmounted(() => graphStore.moduleSource.unobserve(observeSourceChange))
function observeSourceChange(textEdits: readonly SourceRangeEdit[], origin: Origin | undefined) {
// If we received an update from outside the Code Editor while the editor contained uncommitted changes, we cannot
// proceed incrementally; we wait for the changes to be merged as Y.Js AST updates, and then set the view to the
// resulting code.
if (needResync) {
if (!pendingChanges) {
resetView()
needResync = false
}
return
}
// When we aren't in the `needResync` state, we can ignore updates that originated in the Code Editor.
if (origin === 'local:userAction:CodeEditor') return
if (pendingChanges) {
console.info(`Deferring update (editor dirty).`)
needResync = true
return
}
// If none of the above exit-conditions were reached, the transaction is applicable to our current state.
editorView.dispatch({
changes: textEdits.map(textEditToChangeSpec),
annotations: synchronizedModule.of(graphStore.startEdit()),
})
}
// The LS protocol doesn't identify what version of the file updates are in reference to. When diagnostics are received
// from the LS, we map them to the text assuming that they are applicable to the current version of the module. This
// will be correct if there is no one else editing, and we aren't editing faster than the LS can send updates. Typing
// too quickly can result in incorrect ranges, but at idle it should correct itself when we receive new diagnostics.
watch([viewInitialized, () => projectStore.diagnostics], ([ready, diagnostics]) => {
if (!ready) return
executionContextDiagnostics.value =
graphStore.moduleSource.text ?
lsDiagnosticsToCMDiagnostics(graphStore.moduleSource.text, diagnostics)
: []
})
watch([executionContextDiagnostics, expressionUpdatesDiagnostics], () => {
editorView.dispatch({ effects: diagnosticsUpdated.of(null) })
forceLinting(editorView)
})
onMounted(() => {
editorView.focus()
rootElement.value?.prepend(editorView.dom)
// API for e2e tests.
;(window as any).__codeEditorApi = {
textContent: () => editorView.state.doc.toString(),
textLength: () => editorView.state.doc.length,
indexOf: (substring: string, position?: number) =>
editorView.state.doc.toString().indexOf(substring, position),
placeCursor: (at: number) => {
editorView.dispatch({ selection: EditorSelection.create([EditorSelection.cursor(at)]) })
},
select: (from: number, to: number) => {
editorView.dispatch({ selection: EditorSelection.create([EditorSelection.range(from, to)]) })
},
selectAndReplace: (from: number, to: number, replaceWith: string) => {
editorView.dispatch({ selection: EditorSelection.create([EditorSelection.range(from, to)]) })
editorView.dispatch(editorView.state.update(editorView.state.replaceSelection(replaceWith)))
},
writeText: (text: string, from: number) => {
editorView.dispatch({
changes: [{ from: from, insert: text }],
selection: { anchor: from + text.length },
})
},
}
})
</script>
<template>
<EditorRoot ref="editorRoot" class="CodeEditor" />
<Suspense>
<LazyCodeEditor />
</Suspense>
</template>
<style scoped>
.CodeEditor {
font-family: var(--font-mono);
backdrop-filter: var(--blur-app-bg);
background-color: rgba(255, 255, 255, 0.9);
box-shadow: 0 4px 30px rgba(0, 0, 0, 0.1);
border: 1px solid rgba(255, 255, 255, 0.4);
}
:deep(.cm-scroller) {
font-family: var(--font-mono);
/* Prevent touchpad back gesture, which can be triggered while panning. */
overscroll-behavior: none;
}
:deep(.cm-editor) {
position: relative;
width: 100%;
height: 100%;
opacity: 1;
color: black;
text-shadow: 0 0 2px rgba(255, 255, 255, 0.4);
font-size: 12px;
outline: 1px solid transparent;
transition: outline 0.1s ease-in-out;
}
:deep(.cm-focused) {
outline: 1px solid rgba(0, 0, 0, 0.5);
}
:deep(.cm-tooltip-hover) {
padding: 4px;
border-radius: 4px;
border: 1px solid rgba(0, 0, 0, 0.4);
text-shadow: 0 0 2px rgba(255, 255, 255, 0.4);
&::before {
content: '';
background-color: rgba(255, 255, 255, 0.9);
backdrop-filter: blur(64px);
border-radius: 4px;
}
}
:deep(.cm-gutters) {
border-radius: 3px 0 0 3px;
min-width: 32px;
}
</style>

123
app/gui/src/project-view/components/CodeEditor/CodeEditorImpl.vue Normal file
View File

@ -0,0 +1,123 @@
<script setup lang="ts">
import { useEnsoDiagnostics } from '@/components/CodeEditor/diagnostics'
import { ensoSyntax } from '@/components/CodeEditor/ensoSyntax'
import { useEnsoSourceSync } from '@/components/CodeEditor/sync'
import { ensoHoverTooltip } from '@/components/CodeEditor/tooltips'
import EditorRoot from '@/components/codemirror/EditorRoot.vue'
import { testSupport } from '@/components/codemirror/testSupport'
import { useGraphStore } from '@/stores/graph'
import { useProjectStore } from '@/stores/project'
import { useSuggestionDbStore } from '@/stores/suggestionDatabase'
import { useAutoBlur } from '@/util/autoBlur'
import {
bracketMatching,
defaultHighlightStyle,
foldGutter,
syntaxHighlighting,
} from '@codemirror/language'
import { lintGutter } from '@codemirror/lint'
import { highlightSelectionMatches } from '@codemirror/search'
import { EditorState } from '@codemirror/state'
import { EditorView } from '@codemirror/view'
import { type Highlighter } from '@lezer/highlight'
import { minimalSetup } from 'codemirror'
import { computed, onMounted, ref, watch, type ComponentInstance } from 'vue'
const projectStore = useProjectStore()
const graphStore = useGraphStore()
const suggestionDbStore = useSuggestionDbStore()
const editorRoot = ref<ComponentInstance<typeof EditorRoot>>()
const rootElement = computed(() => editorRoot.value?.rootElement)
useAutoBlur(rootElement)
const editorView = new EditorView()
;(window as any).__codeEditorApi = testSupport(editorView)
const { updateListener, connectModuleListener } = useEnsoSourceSync(graphStore, editorView)
const ensoDiagnostics = useEnsoDiagnostics(projectStore, graphStore, editorView)
watch(
() => projectStore.module,
(module) => {
if (!module) return
editorView.setState(
EditorState.create({
extensions: [
minimalSetup,
syntaxHighlighting(defaultHighlightStyle as Highlighter),
bracketMatching(),
foldGutter(),
lintGutter(),
highlightSelectionMatches(),
ensoSyntax(),
updateListener,
ensoHoverTooltip(graphStore, suggestionDbStore),
ensoDiagnostics,
],
}),
)
connectModuleListener()
},
{ immediate: true },
)
onMounted(() => {
editorView.focus()
rootElement.value?.prepend(editorView.dom)
})
</script>
<template>
<EditorRoot ref="editorRoot" class="CodeEditor" />
</template>
<style scoped>
.CodeEditor {
font-family: var(--font-mono);
backdrop-filter: var(--blur-app-bg);
background-color: rgba(255, 255, 255, 0.9);
box-shadow: 0 4px 30px rgba(0, 0, 0, 0.1);
border: 1px solid rgba(255, 255, 255, 0.4);
}
:deep(.cm-scroller) {
font-family: var(--font-mono);
/* Prevent touchpad back gesture, which can be triggered while panning. */
overscroll-behavior: none;
}
:deep(.cm-editor) {
position: relative;
width: 100%;
height: 100%;
opacity: 1;
color: black;
text-shadow: 0 0 2px rgba(255, 255, 255, 0.4);
font-size: 12px;
outline: 1px solid transparent;
transition: outline 0.1s ease-in-out;
}
:deep(.cm-focused) {
outline: 1px solid rgba(0, 0, 0, 0.5);
}
:deep(.cm-tooltip-hover) {
padding: 4px;
border-radius: 4px;
border: 1px solid rgba(0, 0, 0, 0.4);
text-shadow: 0 0 2px rgba(255, 255, 255, 0.4);
&::before {
content: '';
background-color: rgba(255, 255, 255, 0.9);
backdrop-filter: blur(64px);
border-radius: 4px;
}
}
:deep(.cm-gutters) {
border-radius: 3px 0 0 3px;
min-width: 32px;
}
</style>

207
app/gui/src/project-view/components/CodeEditor/codemirror.ts
View File

@ -1,207 +0,0 @@
/**
* @file This module is a collection of codemirror related imports that are intended to be loaded
* asynchronously using a single dynamic import, allowing for code splitting.
*/
export { defaultKeymap } from '@codemirror/commands'
export {
bracketMatching,
defaultHighlightStyle,
foldGutter,
foldNodeProp,
syntaxHighlighting,
} from '@codemirror/language'
export { forceLinting, lintGutter, linter, type Diagnostic } from '@codemirror/lint'
export { highlightSelectionMatches } from '@codemirror/search'
export { Annotation, EditorState, StateEffect, StateField, type ChangeSet } from '@codemirror/state'
export { EditorView, tooltips, type TooltipView } from '@codemirror/view'
export { type Highlighter } from '@lezer/highlight'
export { minimalSetup } from 'codemirror'
export { yCollab } from 'y-codemirror.next'
import { RawAstExtended } from '@/util/ast/extended'
import { RawAst } from '@/util/ast/raw'
import {
Language,
LanguageSupport,
defineLanguageFacet,
foldNodeProp,
languageDataProp,
syntaxTree,
} from '@codemirror/language'
import { type Diagnostic } from '@codemirror/lint'
import type { ChangeSpec } from '@codemirror/state'
import { hoverTooltip as originalHoverTooltip, type TooltipView } from '@codemirror/view'
import {
NodeProp,
NodeSet,
NodeType,
Parser,
Tree,
type Input,
type PartialParse,
type SyntaxNode,
} from '@lezer/common'
import { styleTags, tags } from '@lezer/highlight'
import { EditorView } from 'codemirror'
import * as iter from 'enso-common/src/utilities/data/iter'
import type { Diagnostic as LSDiagnostic } from 'ydoc-shared/languageServerTypes'
import type { SourceRangeEdit } from 'ydoc-shared/util/data/text'
/** TODO: Add docs */
export function lsDiagnosticsToCMDiagnostics(
source: string,
diagnostics: LSDiagnostic[],
): Diagnostic[] {
if (!diagnostics.length) return []
const results: Diagnostic[] = []
let pos = 0
const lineStartIndices = []
for (const line of source.split('\n')) {
lineStartIndices.push(pos)
pos += line.length + 1
}
for (const diagnostic of diagnostics) {
if (!diagnostic.location) continue
const from =
(lineStartIndices[diagnostic.location.start.line] ?? 0) + diagnostic.location.start.character
const to =
(lineStartIndices[diagnostic.location.end.line] ?? 0) + diagnostic.location.end.character
if (to > source.length || from > source.length) {
// Suppress temporary errors if the source is not the version of the document the LS is reporting diagnostics for.
continue
}
const severity =
diagnostic.kind === 'Error' ? 'error'
: diagnostic.kind === 'Warning' ? 'warning'
: 'info'
results.push({ from, to, message: diagnostic.message, severity })
}
return results
}
type AstNode = RawAstExtended<RawAst.Tree | RawAst.Token, false>
const nodeTypes: NodeType[] = [
...RawAst.Tree.typeNames.map((name, id) => NodeType.define({ id, name })),
...RawAst.Token.typeNames.map((name, id) =>
NodeType.define({ id: id + RawAst.Tree.typeNames.length, name: 'Token' + name }),
),
]
const nodeSet = new NodeSet(nodeTypes).extend(
styleTags({
Ident: tags.variableName,
'Private!': tags.variableName,
Number: tags.number,
'Wildcard!': tags.variableName,
'TextLiteral!': tags.string,
OprApp: tags.operator,
TokenOperator: tags.operator,
'Assignment/TokenOperator': tags.definitionOperator,
UnaryOprApp: tags.operator,
'Function/Ident': tags.function(tags.variableName),
ForeignFunction: tags.function(tags.variableName),
'Import/TokenIdent': tags.function(tags.moduleKeyword),
Export: tags.function(tags.moduleKeyword),
Lambda: tags.function(tags.variableName),
Documented: tags.docComment,
ConstructorDefinition: tags.function(tags.variableName),
}),
foldNodeProp.add({
Function: (node) => node.lastChild,
ArgumentBlockApplication: (node) => node,
OperatorBlockApplication: (node) => node,
}),
)
export const astProp = new NodeProp<AstNode>({ perNode: true })
function astToCodeMirrorTree(
nodeSet: NodeSet,
ast: AstNode,
props?: readonly [number | NodeProp<any>, any][] | undefined,
): Tree {
const [start, end] = ast.span()
const children = ast.children()
const childrenToConvert = iter.tryGetSoleValue(children)?.isToken() ? [] : children
const tree = new Tree(
nodeSet.types[ast.inner.type + (ast.isToken() ? RawAst.Tree.typeNames.length : 0)]!,
childrenToConvert.map((child) => astToCodeMirrorTree(nodeSet, child)),
childrenToConvert.map((child) => child.span()[0] - start),
end - start,
[...(props ?? []), [astProp, ast]],
)
return tree
}
const facet = defineLanguageFacet()
class EnsoParser extends Parser {
nodeSet
constructor() {
super()
this.nodeSet = nodeSet
}
cachedCode: string | undefined
cachedTree: Tree | undefined
createParse(input: Input): PartialParse {
return {
parsedPos: input.length,
stopAt: () => {},
stoppedAt: null,
advance: () => {
const code = input.read(0, input.length)
if (code !== this.cachedCode || this.cachedTree == null) {
this.cachedCode = code
const ast = RawAstExtended.parse(code)
this.cachedTree = astToCodeMirrorTree(this.nodeSet, ast, [[languageDataProp, facet]])
}
return this.cachedTree
},
}
}
}
class EnsoLanguage extends Language {
constructor() {
super(facet, new EnsoParser())
}
}
const ensoLanguage = new EnsoLanguage()
/** TODO: Add docs */
export function enso() {
return new LanguageSupport(ensoLanguage)
}
/** TODO: Add docs */
export function hoverTooltip(
create: (
ast: AstNode,
syntax: SyntaxNode,
) => TooltipView | ((view: EditorView) => TooltipView) | null | undefined,
) {
return originalHoverTooltip((view, pos, side) => {
const syntaxNode = syntaxTree(view.state).resolveInner(pos, side)
const astNode = syntaxNode.tree?.prop(astProp)
if (astNode == null) return null
const domOrCreate = create(astNode, syntaxNode)
if (domOrCreate == null) return null
return {
pos: syntaxNode.from,
end: syntaxNode.to,
above: true,
arrow: true,
create: typeof domOrCreate !== 'function' ? () => domOrCreate : domOrCreate,
}
})
}
/** TODO: Add docs */
export function textEditToChangeSpec({ range: [from, to], insert }: SourceRangeEdit): ChangeSpec {
return { from, to, insert }
}

139
app/gui/src/project-view/components/CodeEditor/diagnostics.ts Normal file
View File

@ -0,0 +1,139 @@
import { type GraphStore } from '@/stores/graph'
import { type ProjectStore } from '@/stores/project'
import { type Diagnostic, forceLinting, linter } from '@codemirror/lint'
import { type Extension, StateEffect, StateField } from '@codemirror/state'
import { type EditorView } from '@codemirror/view'
import * as iter from 'enso-common/src/utilities/data/iter'
import { computed, shallowRef, watch } from 'vue'
import { type Diagnostic as LSDiagnostic, type Position } from 'ydoc-shared/languageServerTypes'
const executionContextDiagnostics = shallowRef<Diagnostic[]>([])
// Effect that can be applied to the document to invalidate the linter state.
const diagnosticsUpdated = StateEffect.define()
// State value that is perturbed by any `diagnosticsUpdated` effect.
const diagnosticsVersion = StateField.define({
create: (_state) => 0,
update: (value, transaction) => {
for (const effect of transaction.effects) {
if (effect.is(diagnosticsUpdated)) value += 1
}
return value
},
})
/** Given a text, indexes it and returns a function for converting between different ways of identifying positions. */
function stringPosConverter(text: string) {
let pos = 0
const lineStartIndex: number[] = []
for (const line of text.split('\n')) {
lineStartIndex.push(pos)
pos += line.length + 1
}
const length = text.length
function lineColToIndex({
line,
character,
}: {
line: number
character: number
}): number | undefined {
const startIx = lineStartIndex[line]
if (startIx == null) return
const ix = startIx + character
if (ix > length) return
return ix
}
return { lineColToIndex }
}
/** Convert the Language Server's diagnostics to CodeMirror diagnostics. */
function lsDiagnosticsToCMDiagnostics(
diagnostics: LSDiagnostic[],
lineColToIndex: (lineCol: Position) => number | undefined,
) {
const results: Diagnostic[] = []
for (const diagnostic of diagnostics) {
if (!diagnostic.location) continue
const from = lineColToIndex(diagnostic.location.start)
const to = lineColToIndex(diagnostic.location.end)
if (to == null || from == null) {
// Suppress temporary errors if the source is not the version of the document the LS is reporting diagnostics for.
continue
}
const severity =
diagnostic.kind === 'Error' ? 'error'
: diagnostic.kind === 'Warning' ? 'warning'
: 'info'
results.push({ from, to, message: diagnostic.message, severity })
}
return results
}
/**
* CodeMirror extension providing diagnostics for an Enso module. Provides CodeMirror diagnostics based on dataflow
* errors, and diagnostics the LS provided in an `executionStatus` message.
*/
export function useEnsoDiagnostics(
projectStore: Pick<ProjectStore, 'computedValueRegistry' | 'dataflowErrors' | 'diagnostics'>,
graphStore: Pick<GraphStore, 'moduleSource' | 'db'>,
editorView: EditorView,
): Extension {
const expressionUpdatesDiagnostics = computed(() => {
const updates = projectStore.computedValueRegistry.db
const panics = updates.type.reverseLookup('Panic')
const errors = updates.type.reverseLookup('DataflowError')
const diagnostics: Diagnostic[] = []
for (const externalId of iter.chain(panics, errors)) {
const update = updates.get(externalId)
if (!update) continue
const astId = graphStore.db.idFromExternal(externalId)
if (!astId) continue
const span = graphStore.moduleSource.getSpan(astId)
if (!span) continue
const [from, to] = span
switch (update.payload.type) {
case 'Panic': {
diagnostics.push({ from, to, message: update.payload.message, severity: 'error' })
break
}
case 'DataflowError': {
const error = projectStore.dataflowErrors.lookup(externalId)
if (error?.value?.message) {
diagnostics.push({ from, to, message: error.value.message, severity: 'error' })
}
break
}
}
}
return diagnostics
})
watch([executionContextDiagnostics, expressionUpdatesDiagnostics], () => {
editorView.dispatch({ effects: diagnosticsUpdated.of(null) })
forceLinting(editorView)
})
// The LS protocol doesn't identify what version of the file updates are in reference to. When diagnostics are
// received from the LS, we map them to the text assuming that they are applicable to the current version of the
// module. This will be correct if there is no one else editing, and we aren't editing faster than the LS can send
// updates. Typing too quickly can result in incorrect ranges, but at idle it should correct itself when we receive
// new diagnostics.
watch(
() => projectStore.diagnostics,
(diagnostics) => {
const { lineColToIndex } = stringPosConverter(graphStore.moduleSource.text)
executionContextDiagnostics.value = lsDiagnosticsToCMDiagnostics(diagnostics, lineColToIndex)
},
)
return [
diagnosticsVersion,
linter(() => [...executionContextDiagnostics.value, ...expressionUpdatesDiagnostics.value], {
needsRefresh(update) {
return (
update.state.field(diagnosticsVersion) !== update.startState.field(diagnosticsVersion)
)
},
}),
]
}

116
app/gui/src/project-view/components/CodeEditor/ensoSyntax.ts Normal file
View File

@ -0,0 +1,116 @@
import { RawAstExtended } from '@/util/ast/extended'
import { RawAst } from '@/util/ast/raw'
import {
defineLanguageFacet,
foldNodeProp,
Language,
languageDataProp,
LanguageSupport,
} from '@codemirror/language'
import {
type Input,
NodeProp,
NodeSet,
NodeType,
Parser,
type PartialParse,
Tree,
} from '@lezer/common'
import { styleTags, tags } from '@lezer/highlight'
import * as iter from 'enso-common/src/utilities/data/iter'
const nodeTypes: NodeType[] = [
...RawAst.Tree.typeNames.map((name, id) => NodeType.define({ id, name })),
...RawAst.Token.typeNames.map((name, id) =>
NodeType.define({ id: id + RawAst.Tree.typeNames.length, name: 'Token' + name }),
),
]
const nodeSet = new NodeSet(nodeTypes).extend(
styleTags({
Ident: tags.variableName,
'Private!': tags.variableName,
Number: tags.number,
'Wildcard!': tags.variableName,
'TextLiteral!': tags.string,
OprApp: tags.operator,
TokenOperator: tags.operator,
'Assignment/TokenOperator': tags.definitionOperator,
UnaryOprApp: tags.operator,
'Function/Ident': tags.function(tags.variableName),
ForeignFunction: tags.function(tags.variableName),
'Import/TokenIdent': tags.function(tags.moduleKeyword),
Export: tags.function(tags.moduleKeyword),
Lambda: tags.function(tags.variableName),
Documented: tags.docComment,
ConstructorDefinition: tags.function(tags.variableName),
}),
foldNodeProp.add({
Function: (node) => node.lastChild,
ArgumentBlockApplication: (node) => node,
OperatorBlockApplication: (node) => node,
}),
)
type AstNode = RawAstExtended<RawAst.Tree | RawAst.Token, false>
const astProp = new NodeProp<AstNode>({ perNode: true })
function astToCodeMirrorTree(
nodeSet: NodeSet,
ast: AstNode,
props?: readonly [number | NodeProp<any>, any][] | undefined,
): Tree {
const [start, end] = ast.span()
const children = ast.children()
const childrenToConvert = iter.tryGetSoleValue(children)?.isToken() ? [] : children
return new Tree(
nodeSet.types[ast.inner.type + (ast.isToken() ? RawAst.Tree.typeNames.length : 0)]!,
childrenToConvert.map((child) => astToCodeMirrorTree(nodeSet, child)),
childrenToConvert.map((child) => child.span()[0] - start),
end - start,
[...(props ?? []), [astProp, ast]],
)
}
const facet = defineLanguageFacet()
class EnsoParser extends Parser {
nodeSet
constructor() {
super()
this.nodeSet = nodeSet
}
cachedCode: string | undefined
cachedTree: Tree | undefined
createParse(input: Input): PartialParse {
return {
parsedPos: input.length,
stopAt: () => {},
stoppedAt: null,
advance: () => {
const code = input.read(0, input.length)
if (code !== this.cachedCode || this.cachedTree == null) {
this.cachedCode = code
const ast = RawAstExtended.parse(code)
this.cachedTree = astToCodeMirrorTree(this.nodeSet, ast, [[languageDataProp, facet]])
}
return this.cachedTree
},
}
}
}
class EnsoLanguage extends Language {
constructor() {
super(facet, new EnsoParser())
}
}
const ensoLanguage = new EnsoLanguage()
/** TODO: Add docs */
export function ensoSyntax() {
return new LanguageSupport(ensoLanguage)
}

123
app/gui/src/project-view/components/CodeEditor/sync.ts Normal file
View File

@ -0,0 +1,123 @@
import type { GraphStore } from '@/stores/graph'
import { Annotation, ChangeSet, type ChangeSpec } from '@codemirror/state'
import { EditorView } from '@codemirror/view'
import { createDebouncer } from 'lib0/eventloop'
import { onUnmounted } from 'vue'
import { MutableModule } from 'ydoc-shared/ast'
import { SourceRangeEdit, textChangeToEdits } from 'ydoc-shared/util/data/text'
import type { Origin } from 'ydoc-shared/yjsModel'
function changeSetToTextEdits(changes: ChangeSet) {
const textEdits = new Array<SourceRangeEdit>()
changes.iterChanges((from, to, _fromB, _toB, insert) =>
textEdits.push({ range: [from, to], insert: insert.toString() }),
)
return textEdits
}
function textEditToChangeSpec({ range: [from, to], insert }: SourceRangeEdit): ChangeSpec {
return { from, to, insert }
}
// Indicates a change updating the text to correspond to the given module state.
const synchronizedModule = Annotation.define<MutableModule>()
/** @returns A CodeMirror Extension that synchronizes the editor state with the AST of an Enso module. */
export function useEnsoSourceSync(
graphStore: Pick<GraphStore, 'moduleSource' | 'viewModule' | 'startEdit' | 'commitEdit'>,
editorView: EditorView,
) {
let pendingChanges: ChangeSet | undefined
let currentModule: MutableModule | undefined
const debounceUpdates = createDebouncer(0)
const updateListener = EditorView.updateListener.of((update) => {
for (const transaction of update.transactions) {
const newModule = transaction.annotation(synchronizedModule)
if (newModule) {
// Flush the pipeline of edits that were based on the old module.
commitPendingChanges()
currentModule = newModule
} else if (transaction.docChanged && currentModule) {
pendingChanges =
pendingChanges ? pendingChanges.compose(transaction.changes) : transaction.changes
// Defer the update until after pending events have been processed, so that if changes are arriving faster
// than we would be able to apply them individually we coalesce them to keep up.
debounceUpdates(commitPendingChanges)
}
}
})
/** Set the editor contents the current module state, discarding any pending editor-initiated changes. */
function resetView() {
pendingChanges = undefined
currentModule = undefined
const viewText = editorView.state.doc.toString()
const code = graphStore.moduleSource.text
const changes = textChangeToEdits(viewText, code).map(textEditToChangeSpec)
console.info('Resetting the editor to the module code.', changes)
editorView.dispatch({
changes,
annotations: synchronizedModule.of(graphStore.startEdit()),
})
}
function checkSync() {
const code = graphStore.viewModule.root()?.code() ?? ''
const viewText = editorView.state.doc.toString()
const uncommitted = textChangeToEdits(code, viewText).map(textEditToChangeSpec)
if (uncommitted.length > 0) {
console.warn(`Module source was not synced to editor content\n${code}`, uncommitted)
}
}
/** Apply any pending changes to the currently-synchronized module, clearing the set of pending changes. */
function commitPendingChanges() {
if (!pendingChanges || !currentModule) return
const changes = pendingChanges
pendingChanges = undefined
const edits = changeSetToTextEdits(changes)
try {
currentModule.applyTextEdits(edits, graphStore.viewModule)
graphStore.commitEdit(currentModule, undefined, 'local:userAction:CodeEditor')
checkSync()
} catch (error) {
console.error(`Code Editor failed to modify module`, error)
resetView()
}
}
let needResync = false
function observeSourceChange(textEdits: readonly SourceRangeEdit[], origin: Origin | undefined) {
// If we received an update from outside the Code Editor while the editor contained uncommitted changes, we cannot
// proceed incrementally; we wait for the changes to be merged as Y.Js AST updates, and then set the view to the
// resulting code.
if (needResync) {
if (!pendingChanges) {
resetView()
needResync = false
}
return
}
// When we aren't in the `needResync` state, we can ignore updates that originated in the Code Editor.
if (origin === 'local:userAction:CodeEditor') {
return
}
if (pendingChanges) {
console.info(`Deferring update (editor dirty).`)
needResync = true
return
}
// If none of the above exit-conditions were reached, the transaction is applicable to our current state.
editorView.dispatch({
changes: textEdits.map(textEditToChangeSpec),
annotations: synchronizedModule.of(graphStore.startEdit()),
})
}
onUnmounted(() => graphStore.moduleSource.unobserve(observeSourceChange))
return {
updateListener,
connectModuleListener: () => graphStore.moduleSource.observe(observeSourceChange),
}
}

106
app/gui/src/project-view/components/CodeEditor/tooltips.ts Normal file
View File

@ -0,0 +1,106 @@
import type { GraphStore, NodeId } from '@/stores/graph'
import { type SuggestionDbStore } from '@/stores/suggestionDatabase'
import { type RawAstExtended } from '@/util/ast/extended'
import { RawAst } from '@/util/ast/raw'
import { qnJoin, tryQualifiedName } from '@/util/qualifiedName'
import { syntaxTree } from '@codemirror/language'
import { type Extension } from '@codemirror/state'
import {
type EditorView,
hoverTooltip as originalHoverTooltip,
tooltips,
type TooltipView,
} from '@codemirror/view'
import { NodeProp, type SyntaxNode } from '@lezer/common'
import { unwrap } from 'ydoc-shared/util/data/result'
import { rangeEncloses } from 'ydoc-shared/yjsModel'
type AstNode = RawAstExtended<RawAst.Tree | RawAst.Token, false>
const astProp = new NodeProp<AstNode>({ perNode: true })
/** TODO: Add docs */
function hoverTooltip(
create: (
ast: AstNode,
syntax: SyntaxNode,
) => TooltipView | ((view: EditorView) => TooltipView) | null | undefined,
): Extension {
return [
tooltips({ position: 'absolute' }),
originalHoverTooltip((view, pos, side) => {
const syntaxNode = syntaxTree(view.state).resolveInner(pos, side)
const astNode = syntaxNode.tree?.prop(astProp)
if (astNode == null) return null
const domOrCreate = create(astNode, syntaxNode)
if (domOrCreate == null) return null
return {
pos: syntaxNode.from,
end: syntaxNode.to,
above: true,
arrow: true,
create: typeof domOrCreate !== 'function' ? () => domOrCreate : domOrCreate,
}
}),
]
}
/** @returns A CodeMirror extension that creates tooltips containing type and syntax information for Enso code. */
export function ensoHoverTooltip(
graphStore: Pick<GraphStore, 'moduleSource' | 'db'>,
suggestionDbStore: Pick<SuggestionDbStore, 'entries'>,
) {
return hoverTooltip((ast, syn) => {
const dom = document.createElement('div')
const astSpan = ast.span()
let foundNode: NodeId | undefined
for (const [id, node] of graphStore.db.nodeIdToNode.entries()) {
const rootSpan = graphStore.moduleSource.getSpan(node.rootExpr.id)
if (rootSpan && rangeEncloses(rootSpan, astSpan)) {
foundNode = id
break
}
}
const expressionInfo = foundNode && graphStore.db.getExpressionInfo(foundNode)
const nodeColor = foundNode && graphStore.db.getNodeColorStyle(foundNode)
if (foundNode != null) {
dom
.appendChild(document.createElement('div'))
.appendChild(document.createTextNode(`AST ID: ${foundNode}`))
}
if (expressionInfo != null) {
dom
.appendChild(document.createElement('div'))
.appendChild(document.createTextNode(`Type: ${expressionInfo.typename ?? 'Unknown'}`))
}
if (expressionInfo?.profilingInfo[0] != null) {
const profile = expressionInfo.profilingInfo[0]
const executionTime = (profile.ExecutionTime.nanoTime / 1_000_000).toFixed(3)
const text = `Execution Time: ${executionTime}ms`
dom.appendChild(document.createElement('div')).appendChild(document.createTextNode(text))
}
dom
.appendChild(document.createElement('div'))
.appendChild(document.createTextNode(`Syntax: ${syn.toString()}`))
const method = expressionInfo?.methodCall?.methodPointer
if (method != null) {
const moduleName = tryQualifiedName(method.module)
const methodName = tryQualifiedName(method.name)
const qualifiedName = qnJoin(unwrap(moduleName), unwrap(methodName))
const [id] = suggestionDbStore.entries.nameToId.lookup(qualifiedName)
const suggestionEntry = id != null ? suggestionDbStore.entries.get(id) : undefined
if (suggestionEntry != null) {
const groupNode = dom.appendChild(document.createElement('div'))
groupNode.appendChild(document.createTextNode('Group: '))
const groupNameNode = groupNode.appendChild(document.createElement('span'))
groupNameNode.appendChild(document.createTextNode(`${method.module}.${method.name}`))
if (nodeColor) {
groupNameNode.style.color = nodeColor
}
}
}
return { dom }
})
}

2
app/gui/src/project-view/components/DocumentationEditor.vue
View File

@ -223,7 +223,7 @@ const handler = documentationEditorBindings.handler({
>
<MarkdownEditor
ref="markdownEditor"
:yText="yText"
:content="yText"
:transformImageUrl="transformImageUrl"
:toolbarContainer="toolbarElement"
/>

13
app/gui/src/project-view/components/MarkdownEditor.vue
View File

@ -1,11 +1,14 @@
<script setup lang="ts">
import type { UrlTransformer } from '@/components/MarkdownEditor/imageUrlTransformer'
import {
provideDocumentationImageUrlTransformer,
type UrlTransformer,
} from '@/components/MarkdownEditor/imageUrlTransformer'
import { Vec2 } from '@/util/data/vec2'
import { ComponentInstance, computed, defineAsyncComponent, ref } from 'vue'
import { ComponentInstance, computed, defineAsyncComponent, ref, toRef } from 'vue'
import * as Y from 'yjs'
const props = defineProps<{
yText: Y.Text
content: Y.Text | string
transformImageUrl?: UrlTransformer
toolbarContainer: HTMLElement | undefined
}>()
@ -16,6 +19,8 @@ const LazyMarkdownEditor = defineAsyncComponent(
() => import('@/components/MarkdownEditor/MarkdownEditorImpl.vue'),
)
provideDocumentationImageUrlTransformer(toRef(props, 'transformImageUrl'))
defineExpose({
loaded: computed(() => inner.value != null),
putText: (text: string) => {
@ -29,6 +34,6 @@ defineExpose({
<template>
<Suspense>
<LazyMarkdownEditor ref="inner" v-bind="props" />
<LazyMarkdownEditor ref="inner" v-bind="props" class="MarkdownEditor" />
</Suspense>
</template>

81
app/gui/src/project-view/components/MarkdownEditor/MarkdownEditorImpl.vue
View File

@ -1,55 +1,60 @@
<script setup lang="ts">
import EditorRoot from '@/components/EditorRoot.vue'
import EditorRoot from '@/components/codemirror/EditorRoot.vue'
import { yCollab } from '@/components/codemirror/yCollab'
import { highlightStyle } from '@/components/MarkdownEditor/highlight'
import {
provideDocumentationImageUrlTransformer,
type UrlTransformer,
} from '@/components/MarkdownEditor/imageUrlTransformer'
import { ensoMarkdown } from '@/components/MarkdownEditor/markdown'
import VueComponentHost from '@/components/VueComponentHost.vue'
import { assert } from '@/util/assert'
import { Vec2 } from '@/util/data/vec2'
import { EditorState, Text } from '@codemirror/state'
import { EditorView } from '@codemirror/view'
import { minimalSetup } from 'codemirror'
import { type ComponentInstance, onMounted, ref, toRef, useCssModule, watch } from 'vue'
import { yCollab } from 'y-codemirror.next'
import * as awarenessProtocol from 'y-protocols/awareness.js'
import { type ComponentInstance, computed, onMounted, ref, toRef, useCssModule, watch } from 'vue'
import { Awareness } from 'y-protocols/awareness.js'
import * as Y from 'yjs'
const editorRoot = ref<ComponentInstance<typeof EditorRoot>>()
const props = defineProps<{
yText: Y.Text
transformImageUrl?: UrlTransformer | undefined
toolbarContainer: HTMLElement | undefined
content: Y.Text | string
toolbarContainer?: HTMLElement | undefined
}>()
const vueHost = ref<ComponentInstance<typeof VueComponentHost>>()
const focused = ref(false)
const readonly = computed(() => typeof props.content === 'string')
const editing = computed(() => !readonly.value && focused.value)
provideDocumentationImageUrlTransformer(toRef(props, 'transformImageUrl'))
const awareness = new awarenessProtocol.Awareness(new Y.Doc())
const awareness = new Awareness(new Y.Doc())
const editorView = new EditorView()
// Disable EditContext API because of https://github.com/codemirror/dev/issues/1458.
;(EditorView as any).EDIT_CONTEXT = false
const constantExtensions = [minimalSetup, highlightStyle(useCssModule()), EditorView.lineWrapping]
watch([vueHost, toRef(props, 'yText')], ([vueHost, yText]) => {
watch([vueHost, toRef(props, 'content')], ([vueHost, content]) => {
if (!vueHost) return
editorView.setState(
EditorState.create({
doc: yText.toString(),
extensions: [...constantExtensions, ensoMarkdown({ vueHost }), yCollab(yText, awareness)],
}),
)
let doc = ''
const extensions = [...constantExtensions, ensoMarkdown({ vueHost })]
if (typeof content === 'string') {
doc = content
} else {
assert(content.doc !== null)
const yTextWithDoc: Y.Text & { doc: Y.Doc } = content as any
doc = content.toString()
extensions.push(yCollab(yTextWithDoc, awareness))
}
editorView.setState(EditorState.create({ doc, extensions }))
})
onMounted(() => {
const content = editorView.dom.getElementsByClassName('cm-content')[0]!
content.addEventListener('focusin', () => (editing.value = true))
// Enable rendering the line containing the current cursor in `editing` mode if focus enters the element *inside* the
// scroll area--if we attached the handler to the editor root, clicking the scrollbar would cause editing mode to be
// activated.
editorView.dom
.getElementsByClassName('cm-content')[0]!
.addEventListener('focusin', () => (focused.value = true))
editorRoot.value?.rootElement?.prepend(editorView.dom)
})
const editing = ref(false)
/**
* Replace text in given document range with `text`, putting text cursor after inserted text.
*
@ -77,12 +82,7 @@ defineExpose({
</script>
<template>
<EditorRoot
ref="editorRoot"
class="MarkdownEditor"
:class="{ editing }"
@focusout="editing = false"
/>
<EditorRoot ref="editorRoot" v-bind="$attrs" :class="{ editing }" @focusout="focused = false" />
<VueComponentHost ref="vueHost" />
</template>
@ -91,24 +91,15 @@ defineExpose({
font-family: var(--font-sans);
}
:deep(.cm-scroller) {
/* Prevent touchpad back gesture, which can be triggered while panning. */
overscroll-behavior: none;
:deep(.cm-editor) {
opacity: 1;
color: black;
font-size: 12px;
}
:deep(img.uploading) {
opacity: 0.5;
}
.EditorRoot :deep(.cm-editor) {
position: relative;
width: 100%;
height: 100%;
opacity: 1;
color: black;
font-size: 12px;
outline: none;
}
</style>
<!--suppress CssUnusedSymbol -->

74
app/gui/src/project-view/components/MarkdownEditor/TableEditor.vue Normal file
View File

@ -0,0 +1,74 @@
<script setup lang="ts">
import MarkdownEditorImpl from '@/components/MarkdownEditor/MarkdownEditorImpl.vue'
import type { Text } from '@codemirror/state'
import { SyntaxNode, TreeCursor } from '@lezer/common'
import { computed } from 'vue'
const { source, parsed } = defineProps<{
source: Text
parsed: SyntaxNode
}>()
function parseRow(cursor: TreeCursor, output: string[]) {
if (!cursor.firstChild()) return
do {
if (cursor.name === 'TableCell') {
output.push(source.sliceString(cursor.from, cursor.to))
} else if (cursor.name !== 'TableDelimiter') {
console.warn('Unexpected in table row:', cursor.name)
}
} while (cursor.nextSibling())
cursor.parent()
}
const content = computed(() => {
const headers: string[] = []
const rows: string[][] = []
const cursor = parsed.cursor()
if (cursor.firstChild()) {
do {
if (cursor.name === 'TableRow') {
const newRow: string[] = []
parseRow(cursor, newRow)
rows.push(newRow)
} else if (cursor.name === 'TableHeader') {
parseRow(cursor, headers)
} else if (cursor.name !== 'TableDelimiter') {
console.warn('Unexpected at top level of table:', cursor.name)
}
} while (cursor.nextSibling())
}
return { headers, rows }
})
</script>
<template>
<table>
<thead>
<tr>
<th v-for="(cell, c) in content.headers" :key="c" class="cell">
<MarkdownEditorImpl :content="cell" />
</th>
</tr>
</thead>
<tbody class="tableBody">
<tr v-for="(row, r) in content.rows" :key="r" class="row">
<td v-for="(cell, c) in row" :key="c" class="cell">
<MarkdownEditorImpl :content="cell" />
</td>
</tr>
</tbody>
</table>
</template>
<style scoped>
.cell {
border: 1px solid #dddddd;
}
.tableBody .row:nth-of-type(even) {
background-color: #f3f3f3;
}
:deep(.cm-line) {
padding-right: 6px;
}
</style>

95
app/gui/src/project-view/components/MarkdownEditor/markdown.ts
View File

@ -1,9 +1,98 @@
import { markdownDecorators } from '@/components/MarkdownEditor/markdown/decoration'
import { markdown } from '@/components/MarkdownEditor/markdown/parse'
import type { VueHost } from '@/components/VueComponentHost.vue'
import { markdown as markdownExtension } from '@codemirror/lang-markdown'
import {
defineLanguageFacet,
foldNodeProp,
foldService,
indentNodeProp,
Language,
languageDataProp,
syntaxTree,
} from '@codemirror/language'
import type { Extension } from '@codemirror/state'
import { NodeProp, type NodeType, type Parser, type SyntaxNode } from '@lezer/common'
import { markdownParser } from 'ydoc-shared/ast/ensoMarkdown'
/** Markdown extension, with customizations for Enso. */
/** CodeMirror Extension for the Enso Markdown dialect. */
export function ensoMarkdown({ vueHost }: { vueHost: VueHost }): Extension {
return [markdown(), markdownDecorators({ vueHost })]
return [
markdownExtension({
base: mkLang(
markdownParser.configure([
commonmarkCodemirrorLanguageExtension,
tableCodemirrorLanguageExtension,
]),
),
}),
markdownDecorators({ vueHost }),
]
}
function mkLang(parser: Parser) {
return new Language(data, parser, [headerIndent], 'markdown')
}
const data = defineLanguageFacet({ commentTokens: { block: { open: '<!--', close: '-->' } } })
const headingProp = new NodeProp<number>()
const commonmarkCodemirrorLanguageExtension = {
props: [
foldNodeProp.add((type) => {
return !type.is('Block') || type.is('Document') || isHeading(type) != null || isList(type) ?
undefined
: (tree, state) => ({ from: state.doc.lineAt(tree.from).to, to: tree.to })
}),
headingProp.add(isHeading),
indentNodeProp.add({
Document: () => null,
}),
languageDataProp.add({
Document: data,
}),
],
}
function isHeading(type: NodeType) {
const match = /^(?:ATX|Setext)Heading(\d)$/.exec(type.name)
return match ? +match[1]! : undefined
}
function isList(type: NodeType) {
return type.name == 'OrderedList' || type.name == 'BulletList'
}
function findSectionEnd(headerNode: SyntaxNode, level: number) {
let last = headerNode
for (;;) {
const next = last.nextSibling
let heading
if (!next || ((heading = isHeading(next.type)) != null && heading <= level)) break
last = next
}
return last.to
}
const headerIndent = foldService.of((state, start, end) => {
for (
let node: SyntaxNode | null = syntaxTree(state).resolveInner(end, -1);
node;
node = node.parent
) {
if (node.from < start) break
const heading = node.type.prop(headingProp)
if (heading == null) continue
const upto = findSectionEnd(node, heading)
if (upto > end) return { from: end, to: upto }
}
return null
})
const tableCodemirrorLanguageExtension = {
props: [
foldNodeProp.add({
Table: (tree, state) => ({ from: state.doc.lineAt(tree.from).to, to: tree.to }),
}),
],
}

69
app/gui/src/project-view/components/MarkdownEditor/markdown/decoration.ts
View File

@ -1,4 +1,5 @@
import DocumentationImage from '@/components/MarkdownEditor/DocumentationImage.vue'
import TableEditor from '@/components/MarkdownEditor/TableEditor.vue'
import type { VueHost } from '@/components/VueComponentHost.vue'
import { syntaxTree } from '@codemirror/language'
import { type EditorSelection, type Extension, RangeSetBuilder, type Text } from '@codemirror/state'
@ -19,6 +20,7 @@ export function markdownDecorators({ vueHost }: { vueHost: VueHost }): Extension
const stateDecorator = new TreeStateDecorator(vueHost, [
decorateImageWithClass,
decorateImageWithRendered,
decorateTable,
])
const stateDecoratorExt = EditorView.decorations.compute(['doc'], (state) =>
stateDecorator.decorate(syntaxTree(state), state.doc),
@ -146,8 +148,6 @@ function parseLinkLike(node: SyntaxNode, doc: Text) {
if (!textClose) return
const urlNode = findNextSiblingNamed(textClose, 'URL')
if (!urlNode) return
console.log('RANGE', urlNode.from, urlNode.to)
console.log(doc)
return {
textFrom: textOpen.to,
textTo: textClose.from,
@ -274,3 +274,68 @@ function findNextSiblingNamed(node: SyntaxNode, name: string) {
}
}
}
// === Tables ===
function decorateTable(
nodeRef: SyntaxNodeRef,
doc: Text,
emitDecoration: (from: number, to: number, deco: Decoration) => void,
vueHost: VueHost,
) {
if (nodeRef.name === 'Table') {
const source = doc //.slice(nodeRef.from, nodeRef.to)
const parsed = nodeRef.node
const widget = new TableWidget({ source, parsed }, vueHost)
emitDecoration(
nodeRef.from,
nodeRef.to,
Decoration.replace({
widget,
// Ensure the cursor is drawn relative to the content before the widget.
// If it is drawn relative to the widget, it will be hidden when the widget is hidden (i.e. during editing).
side: 1,
block: true,
}),
)
}
}
class TableWidget extends WidgetType {
private container: HTMLElement | undefined
private vueHostRegistration: { unregister: () => void } | undefined
constructor(
private readonly props: { source: Text; parsed: SyntaxNode },
private readonly vueHost: VueHost,
) {
super()
}
override get estimatedHeight() {
return -1
}
override toDOM(): HTMLElement {
if (!this.container) {
const container = markRaw(document.createElement('div'))
container.className = 'cm-table-editor'
this.vueHostRegistration = this.vueHost.register(
() =>
h(TableEditor, {
source: this.props.source,
parsed: this.props.parsed,
onEdit: () => console.log('onEdit'),
}),
container,
)
this.container = container
}
return this.container
}
override destroy() {
this.vueHostRegistration?.unregister()
this.container = undefined
}
}

12
app/gui/src/project-view/components/EditorRoot.vue → app/gui/src/project-view/components/codemirror/EditorRoot.vue
View File

@ -26,4 +26,16 @@ defineExpose({ rootElement })
width: 100%;
height: 100%;
}
:deep(.cm-scroller) {
/* Prevent touchpad back gesture, which can be triggered while panning. */
overscroll-behavior: none;
}
:deep(.cm-editor) {
position: relative;
width: 100%;
height: 100%;
outline: none;
}
</style>

28
app/gui/src/project-view/components/codemirror/testSupport.ts Normal file
View File

@ -0,0 +1,28 @@
import { EditorSelection } from '@codemirror/state'
import { type EditorView } from '@codemirror/view'
/** Returns an API for the editor content, used by the integration tests. */
export function testSupport(editorView: EditorView) {
return {
textContent: () => editorView.state.doc.toString(),
textLength: () => editorView.state.doc.length,
indexOf: (substring: string, position?: number) =>
editorView.state.doc.toString().indexOf(substring, position),
placeCursor: (at: number) => {
editorView.dispatch({ selection: EditorSelection.create([EditorSelection.cursor(at)]) })
},
select: (from: number, to: number) => {
editorView.dispatch({ selection: EditorSelection.create([EditorSelection.range(from, to)]) })
},
selectAndReplace: (from: number, to: number, replaceWith: string) => {
editorView.dispatch({ selection: EditorSelection.create([EditorSelection.range(from, to)]) })
editorView.dispatch(editorView.state.update(editorView.state.replaceSelection(replaceWith)))
},
writeText: (text: string, from: number) => {
editorView.dispatch({
changes: [{ from: from, insert: text }],
selection: { anchor: from + text.length },
})
},
}
}

65
app/gui/src/project-view/components/codemirror/yCollab/index.ts Normal file
View File

@ -0,0 +1,65 @@
/**
* @file CodeMirror extension for synchronizing with a Yjs Text object.
* Based on <https://github.com/yjs/y-codemirror.next>. Initial changes from upstream:
* - Translated from JSDoc-typed JS to Typescript.
* - Refactored for stricter typing.
* - Changes to match project code style.
*/
import * as cmView from '@codemirror/view'
import { type Awareness } from 'y-protocols/awareness.js'
import * as Y from 'yjs'
import { YRange } from './y-range'
import { yRemoteSelections, yRemoteSelectionsTheme } from './y-remote-selections'
import { YSyncConfig, ySync, ySyncAnnotation, ySyncFacet } from './y-sync'
import {
YUndoManagerConfig,
redo,
undo,
yUndoManager,
yUndoManagerFacet,
yUndoManagerKeymap,
} from './y-undomanager'
export {
YRange,
YSyncConfig,
yRemoteSelections,
yRemoteSelectionsTheme,
ySync,
ySyncAnnotation,
ySyncFacet,
yUndoManagerKeymap,
}
/* CodeMirror Extension for synchronizing the editor state with a {@link Y.Text}. */
export const yCollab = (
ytext: Y.Text & { doc: Y.Doc },
awareness: Awareness | null,
{
undoManager = new Y.UndoManager(ytext),
}: {
/** Set to false to disable the undo-redo plugin */
undoManager?: Y.UndoManager | false
} = {},
) => {
const ySyncConfig = new YSyncConfig(ytext, awareness)
const plugins = [ySyncFacet.of(ySyncConfig), ySync]
if (awareness) {
plugins.push(yRemoteSelectionsTheme, yRemoteSelections)
}
if (undoManager !== false) {
// By default, only track changes that are produced by the sync plugin (local edits)
plugins.push(
yUndoManagerFacet.of(new YUndoManagerConfig(undoManager)),
yUndoManager,
cmView.EditorView.domEventHandlers({
beforeinput(e, view) {
if (e.inputType === 'historyUndo') return undo(view)
if (e.inputType === 'historyRedo') return redo(view)
return false
},
}),
)
}
return plugins
}

32
app/gui/src/project-view/components/codemirror/yCollab/y-range.ts Normal file
View File

@ -0,0 +1,32 @@
import * as Y from 'yjs'
/**
* Defines a range on text using relative positions that can be transformed back to
* absolute positions. (https://docs.yjs.dev/api/relative-positions)
*/
export class YRange {
/** TODO: Add docs */
constructor(
readonly yanchor: Y.RelativePosition,
readonly yhead: Y.RelativePosition,
) {
this.yanchor = yanchor
this.yhead = yhead
}
/** TODO: Add docs */
toJSON() {
return {
yanchor: Y.relativePositionToJSON(this.yanchor),
yhead: Y.relativePositionToJSON(this.yhead),
}
}
/** TODO: Add docs */
static fromJSON(json: { yanchor: unknown; yhead: unknown }) {
return new YRange(
Y.createRelativePositionFromJSON(json.yanchor),
Y.createRelativePositionFromJSON(json.yhead),
)
}
}

264
app/gui/src/project-view/components/codemirror/yCollab/y-remote-selections.ts Normal file
View File

@ -0,0 +1,264 @@
import * as cmState from '@codemirror/state'
import * as cmView from '@codemirror/view'
import * as dom from 'lib0/dom'
import * as math from 'lib0/math'
import * as pair from 'lib0/pair'
import { Awareness } from 'y-protocols/awareness.js'
import { assert } from 'ydoc-shared/util/assert'
import * as Y from 'yjs'
import { type YSyncConfig, ySyncFacet } from './y-sync'
export const yRemoteSelectionsTheme = cmView.EditorView.baseTheme({
'.cm-ySelection': {},
'.cm-yLineSelection': {
padding: 0,
margin: '0px 2px 0px 4px',
},
'.cm-ySelectionCaret': {
position: 'relative',
borderLeft: '1px solid black',
borderRight: '1px solid black',
marginLeft: '-1px',
marginRight: '-1px',
boxSizing: 'border-box',
display: 'inline',
},
'.cm-ySelectionCaretDot': {
borderRadius: '50%',
position: 'absolute',
width: '.4em',
height: '.4em',
top: '-.2em',
left: '-.2em',
backgroundColor: 'inherit',
transition: 'transform .3s ease-in-out',
boxSizing: 'border-box',
},
'.cm-ySelectionCaret:hover > .cm-ySelectionCaretDot': {
transformOrigin: 'bottom center',
transform: 'scale(0)',
},
'.cm-ySelectionInfo': {
position: 'absolute',
top: '-1.05em',
left: '-1px',
fontSize: '.75em',
fontFamily: 'serif',
fontStyle: 'normal',
fontWeight: 'normal',
lineHeight: 'normal',
userSelect: 'none',
color: 'white',
paddingLeft: '2px',
paddingRight: '2px',
zIndex: 101,
transition: 'opacity .3s ease-in-out',
backgroundColor: 'inherit',
// these should be separate
opacity: 0,
transitionDelay: '0s',
whiteSpace: 'nowrap',
},
'.cm-ySelectionCaret:hover > .cm-ySelectionInfo': {
opacity: 1,
transitionDelay: '0s',
},
})
/**
* @todo specify the users that actually changed. Currently, we recalculate positions for every user.
*/
const yRemoteSelectionsAnnotation = cmState.Annotation.define<number[]>()
class YRemoteCaretWidget extends cmView.WidgetType {
constructor(
readonly color: string,
readonly name: string,
) {
super()
}
toDOM() {
return dom.element(
'span',
[
pair.create('class', 'cm-ySelectionCaret'),
pair.create('style', `background-color: ${this.color}; border-color: ${this.color}`),
],
[
dom.text('\u2060'),
dom.element('div', [pair.create('class', 'cm-ySelectionCaretDot')]),
dom.text('\u2060'),
dom.element('div', [pair.create('class', 'cm-ySelectionInfo')], [dom.text(this.name)]),
dom.text('\u2060'),
],
) as HTMLElement
}
override eq(widget: unknown) {
assert(widget instanceof YRemoteCaretWidget)
return widget.color === this.color
}
compare(widget: unknown) {
assert(widget instanceof YRemoteCaretWidget)
return widget.color === this.color
}
override updateDOM() {
return false
}
override get estimatedHeight() {
return -1
}
override ignoreEvent() {
return true
}
}
/** TODO: Add docs */
export class YRemoteSelectionsPluginValue {
private readonly conf: YSyncConfig
private readonly _awareness: Awareness
decorations: cmView.DecorationSet
private readonly _listener: ({ added, updated, removed }: any) => void
/** TODO: Add docs */
constructor(view: cmView.EditorView) {
this.conf = view.state.facet(ySyncFacet)
assert(this.conf.awareness != null)
this._listener = ({ added, updated, removed }: any) => {
const clients = added.concat(updated).concat(removed)
if (clients.findIndex((id: any) => id !== this._awareness.doc.clientID) >= 0) {
view.dispatch({ annotations: [yRemoteSelectionsAnnotation.of([])] })
}
}
this._awareness = this.conf.awareness
this._awareness.on('change', this._listener)
this.decorations = cmState.RangeSet.of([])
}
/** TODO: Add docs */
destroy() {
this._awareness.off('change', this._listener)
}
/** TODO: Add docs */
update(update: cmView.ViewUpdate) {
const ytext = this.conf.ytext
const ydoc = ytext.doc
const awareness = this._awareness
const decorations: cmState.Range<cmView.Decoration>[] = []
const localAwarenessState = this._awareness.getLocalState()
// set local awareness state (update cursors)
if (localAwarenessState != null) {
const hasFocus = update.view.hasFocus && update.view.dom.ownerDocument.hasFocus()
const sel = hasFocus ? update.state.selection.main : null
const currentAnchor =
localAwarenessState.cursor == null ?
null
: Y.createRelativePositionFromJSON(localAwarenessState.cursor.anchor)
const currentHead =
localAwarenessState.cursor == null ?
null
: Y.createRelativePositionFromJSON(localAwarenessState.cursor.head)
if (sel != null) {
const anchor = Y.createRelativePositionFromTypeIndex(ytext, sel.anchor)
const head = Y.createRelativePositionFromTypeIndex(ytext, sel.head)
if (
localAwarenessState.cursor == null ||
!Y.compareRelativePositions(currentAnchor, anchor) ||
!Y.compareRelativePositions(currentHead, head)
) {
awareness.setLocalStateField('cursor', {
anchor,
head,
})
}
} else if (localAwarenessState.cursor != null && hasFocus) {
awareness.setLocalStateField('cursor', null)
}
}
// update decorations (remote selections)
awareness.getStates().forEach((state, clientid) => {
if (clientid === awareness.doc.clientID) {
return
}
const cursor = state.cursor
if (cursor == null || cursor.anchor == null || cursor.head == null) {
return
}
const anchor = Y.createAbsolutePositionFromRelativePosition(cursor.anchor, ydoc)
const head = Y.createAbsolutePositionFromRelativePosition(cursor.head, ydoc)
if (anchor == null || head == null || anchor.type !== ytext || head.type !== ytext) {
return
}
const { color = '#30bced', name = 'Anonymous' } = state.user || {}
const colorLight = (state.user && state.user.colorLight) || color + '33'
const start = math.min(anchor.index, head.index)
const end = math.max(anchor.index, head.index)
const startLine = update.view.state.doc.lineAt(start)
const endLine = update.view.state.doc.lineAt(end)
if (startLine.number === endLine.number) {
// selected content in a single line.
decorations.push({
from: start,
to: end,
value: cmView.Decoration.mark({
attributes: { style: `background-color: ${colorLight}` },
class: 'cm-ySelection',
}),
})
} else {
// selected content in multiple lines
// first, render text-selection in the first line
decorations.push({
from: start,
to: startLine.from + startLine.length,
value: cmView.Decoration.mark({
attributes: { style: `background-color: ${colorLight}` },
class: 'cm-ySelection',
}),
})
// render text-selection in the last line
decorations.push({
from: endLine.from,
to: end,
value: cmView.Decoration.mark({
attributes: { style: `background-color: ${colorLight}` },
class: 'cm-ySelection',
}),
})
for (let i = startLine.number + 1; i < endLine.number; i++) {
const linePos = update.view.state.doc.line(i).from
decorations.push({
from: linePos,
to: linePos,
value: cmView.Decoration.line({
attributes: { style: `background-color: ${colorLight}`, class: 'cm-yLineSelection' },
}),
})
}
}
decorations.push({
from: head.index,
to: head.index,
value: cmView.Decoration.widget({
side: head.index - anchor.index > 0 ? -1 : 1, // the local cursor should be rendered outside the remote selection
block: false,
widget: new YRemoteCaretWidget(color, name),
}),
})
})
this.decorations = cmView.Decoration.set(decorations, true)
}
}
export const yRemoteSelections = cmView.ViewPlugin.fromClass(YRemoteSelectionsPluginValue, {
decorations: (v) => v.decorations,
})

156
app/gui/src/project-view/components/codemirror/yCollab/y-sync.ts Normal file
View File

@ -0,0 +1,156 @@
import * as cmState from '@codemirror/state'
import * as cmView from '@codemirror/view'
import { type Awareness } from 'y-protocols/awareness.js'
import { assertDefined } from 'ydoc-shared/util/assert'
import * as Y from 'yjs'
import { YRange } from './y-range'
/** TODO: Add docs */
export class YSyncConfig {
readonly undoManager: Y.UndoManager
readonly ytext: Y.Text & { doc: Y.Doc }
/** TODO: Add docs */
constructor(
ytext: Y.Text & { doc: Y.Doc },
readonly awareness: Awareness | null,
) {
this.ytext = ytext as Y.Text & { doc: Y.Doc }
this.undoManager = new Y.UndoManager(ytext)
}
/**
* Helper function to transform an absolute index position to a Yjs-based relative position
* (https://docs.yjs.dev/api/relative-positions).
*
* A relative position can be transformed back to an absolute position even after the document has changed. The position is
* automatically adapted. This does not require any position transformations. Relative positions are computed based on
* the internal Yjs document model. Peers that share content through Yjs are guaranteed that their positions will always
* synced up when using relatve positions.
*
* ```js
* import { ySyncFacet } from 'y-codemirror'
*
* ..
* const ysync = view.state.facet(ySyncFacet)
* // transform an absolute index position to a ypos
* const ypos = ysync.getYPos(3)
* // transform the ypos back to an absolute position
* ysync.fromYPos(ypos) // => 3
* ```
*
* It cannot be guaranteed that absolute index positions can be synced up between peers.
* This might lead to undesired behavior when implementing features that require that all peers see the
* same marked range (e.g. a comment plugin).
*/
toYPos(pos: number, assoc = 0) {
return Y.createRelativePositionFromTypeIndex(this.ytext, pos, assoc)
}
/** TODO: Add docs */
fromYPos(rpos: Y.RelativePosition | object) {
const pos = Y.createAbsolutePositionFromRelativePosition(
Y.createRelativePositionFromJSON(rpos),
this.ytext.doc,
)
if (pos == null || pos.type !== this.ytext) {
throw new Error(
'[y-codemirror] The position you want to retrieve was created by a different document',
)
}
return {
pos: pos.index,
assoc: pos.assoc,
}
}
/** TODO: Add docs */
toYRange(range: cmState.SelectionRange) {
const assoc = range.assoc
const yanchor = this.toYPos(range.anchor, assoc)
const yhead = this.toYPos(range.head, assoc)
return new YRange(yanchor, yhead)
}
/** TODO: Add docs */
fromYRange(yrange: YRange) {
const anchor = this.fromYPos(yrange.yanchor)
const head = this.fromYPos(yrange.yhead)
if (anchor.pos === head.pos) {
return cmState.EditorSelection.cursor(head.pos, head.assoc)
}
return cmState.EditorSelection.range(anchor.pos, head.pos)
}
}
export const ySyncFacet = cmState.Facet.define<YSyncConfig, YSyncConfig>({
combine(inputs) {
return inputs[inputs.length - 1]!
},
})
export const ySyncAnnotation = cmState.Annotation.define<YSyncConfig>()
class YSyncPluginValue implements cmView.PluginValue {
private readonly _ytext: Y.Text & { doc: Y.Doc }
private readonly conf: YSyncConfig
private readonly _observer: (event: Y.YTextEvent, tr: Y.Transaction) => void
constructor(private readonly view: cmView.EditorView) {
this.conf = view.state.facet(ySyncFacet)
this._observer = (event: Y.YTextEvent, tr: Y.Transaction) => {
if (tr.origin !== this.conf) {
const delta = event.delta
const changes: { from: number; to: number; insert: string }[] = []
let pos = 0
for (const d of delta) {
if (d.insert != null) {
changes.push({ from: pos, to: pos, insert: d.insert as any })
} else if (d.delete != null) {
changes.push({ from: pos, to: pos + d.delete, insert: '' })
pos += d.delete
} else {
assertDefined(d.retain)
pos += d.retain
}
}
view.dispatch({ changes, annotations: [ySyncAnnotation.of(this.conf)] })
}
}
this._ytext = this.conf.ytext
this._ytext.observe(this._observer)
}
update(update: cmView.ViewUpdate) {
if (
!update.docChanged ||
(update.transactions.length > 0 &&
update.transactions[0]!.annotation(ySyncAnnotation) === this.conf)
) {
return
}
const ytext = this.conf.ytext
ytext.doc.transact(() => {
/**
* This variable adjusts the fromA position to the current position in the Y.Text type.
*/
let adj = 0
update.changes.iterChanges((fromA, toA, fromB, toB, insert) => {
const insertText = insert.sliceString(0, insert.length, '\n')
if (fromA !== toA) {
ytext.delete(fromA + adj, toA - fromA)
}
if (insertText.length > 0) {
ytext.insert(fromA + adj, insertText)
}
adj += insertText.length - (toA - fromA)
})
}, this.conf)
}
destroy() {
this._ytext.unobserve(this._observer)
}
}
export const ySync = cmView.ViewPlugin.fromClass(YSyncPluginValue)

138
app/gui/src/project-view/components/codemirror/yCollab/y-undomanager.ts Normal file
View File

@ -0,0 +1,138 @@
import { type StackItemEvent } from '@/components/codemirror/yCollab/yjsTypes'
import * as cmState from '@codemirror/state'
import * as cmView from '@codemirror/view'
import { createMutex } from 'lib0/mutex'
import * as Y from 'yjs'
import { type YRange } from './y-range'
import { ySyncAnnotation, type YSyncConfig, ySyncFacet } from './y-sync'
/** TODO: Add docs */
export class YUndoManagerConfig {
/** TODO: Add docs */
constructor(readonly undoManager: Y.UndoManager) {}
/** TODO: Add docs */
addTrackedOrigin(origin: unknown) {
this.undoManager.addTrackedOrigin(origin)
}
/** TODO: Add docs */
removeTrackedOrigin(origin: unknown) {
this.undoManager.removeTrackedOrigin(origin)
}
/**
* @returns Whether a change was undone.
*/
undo(): boolean {
return this.undoManager.undo() != null
}
/**
* @returns Whether a change was redone.
*/
redo(): boolean {
return this.undoManager.redo() != null
}
}
export const yUndoManagerFacet = cmState.Facet.define<YUndoManagerConfig, YUndoManagerConfig>({
combine(inputs) {
return inputs[inputs.length - 1]!
},
})
export const yUndoManagerAnnotation = cmState.Annotation.define<YUndoManagerConfig>()
class YUndoManagerPluginValue implements cmView.PluginValue {
private readonly conf: YUndoManagerConfig
private readonly syncConf: YSyncConfig
private _beforeChangeSelection: null | YRange
private readonly _undoManager: Y.UndoManager
private readonly _mux: (cb: () => void, elseCb?: (() => void) | undefined) => any
private readonly _storeSelection: () => void
private readonly _onStackItemAdded: (event: StackItemEvent) => void
private readonly _onStackItemPopped: (event: StackItemEvent) => void
constructor(readonly view: cmView.EditorView) {
this.conf = view.state.facet(yUndoManagerFacet)
this._undoManager = this.conf.undoManager
this.syncConf = view.state.facet(ySyncFacet)
this._beforeChangeSelection = null
this._mux = createMutex()
this._onStackItemAdded = ({ stackItem, changedParentTypes }: StackItemEvent) => {
// only store metadata if this type was affected
if (
changedParentTypes.has(this.syncConf.ytext as any) &&
this._beforeChangeSelection &&
!stackItem.meta.has(this)
) {
// do not overwrite previous stored selection
stackItem.meta.set(this, this._beforeChangeSelection)
}
}
this._onStackItemPopped = ({ stackItem }: StackItemEvent) => {
const sel = stackItem.meta.get(this)
if (sel) {
const selection = this.syncConf.fromYRange(sel)
view.dispatch(
view.state.update({
selection,
effects: [cmView.EditorView.scrollIntoView(selection)],
}),
)
this._storeSelection()
}
}
/**
* Do this without mutex, simply use the sync annotation
*/
this._storeSelection = () => {
// store the selection before the change is applied so we can restore it with the undo manager.
this._beforeChangeSelection = this.syncConf.toYRange(this.view.state.selection.main)
}
this._undoManager.on('stack-item-added', this._onStackItemAdded)
this._undoManager.on('stack-item-popped', this._onStackItemPopped)
this._undoManager.addTrackedOrigin(this.syncConf)
}
update(update: cmView.ViewUpdate) {
if (
update.selectionSet &&
(update.transactions.length === 0 ||
update.transactions[0]!.annotation(ySyncAnnotation) !== this.syncConf)
) {
// This only works when YUndoManagerPlugin is included before the sync plugin
this._storeSelection()
}
}
destroy() {
this._undoManager.off('stack-item-added', this._onStackItemAdded)
this._undoManager.off('stack-item-popped', this._onStackItemPopped)
this._undoManager.removeTrackedOrigin(this.syncConf)
}
}
export const yUndoManager = cmView.ViewPlugin.fromClass(YUndoManagerPluginValue)
export const undo: cmState.StateCommand = ({ state }) =>
state.facet(yUndoManagerFacet).undo() || true
export const redo: cmState.StateCommand = ({ state }) =>
state.facet(yUndoManagerFacet).redo() || true
export const undoDepth = (state: cmState.EditorState): number =>
state.facet(yUndoManagerFacet).undoManager.undoStack.length
export const redoDepth = (state: cmState.EditorState): number =>
state.facet(yUndoManagerFacet).undoManager.redoStack.length
/**
* Default key bindings for the undo manager.
*/
export const yUndoManagerKeymap: cmView.KeyBinding[] = [
{ key: 'Mod-z', run: undo, preventDefault: true },
{ key: 'Mod-y', mac: 'Mod-Shift-z', run: redo, preventDefault: true },
{ key: 'Mod-Shift-z', run: redo, preventDefault: true },
]

28
app/gui/src/project-view/components/codemirror/yCollab/yjsTypes.d.ts vendored Normal file
View File

@ -0,0 +1,28 @@
/** @file Types exposed by Yjs APIs, but not exported by name. */
import * as Y from 'yjs'
export interface StackItemEvent {
stackItem: StackItem
origin: unknown
type: 'undo' | 'redo'
changedParentTypes: Map<Y.AbstractType<Y.YEvent<any>>, Y.YEvent<any>[]>
}
export interface StackItem {
insertions: DeleteSet
deletions: DeleteSet
/**
* Use this to save and restore metadata like selection range
*/
meta: Map<any, any>
}
export interface DeleteSet {
clients: Map<number, DeleteItem[]>
}
export interface DeleteItem {
clock: number
len: number
}

1
app/gui/src/project-view/stores/project/executionContext.ts
View File

@ -9,7 +9,6 @@ import {
type Identifier,
} from '@/util/qualifiedName'
import * as array from 'lib0/array'
import * as object from 'lib0/object'
import { ObservableV2 } from 'lib0/observable'
import * as random from 'lib0/random'
import { reactive } from 'vue'

22
app/licenses/MIT-yCollab-LICENSE Normal file
View File

@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) 2024
- Kevin Jahns <kevin.jahns@protonmail.com>.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

2
app/ydoc-shared/package.json
View File

@ -35,6 +35,8 @@
},
"dependencies": {
"enso-common": "workspace:*",
"@lezer/common": "^1.1.0",
"@lezer/markdown": "^1.3.1",
"@noble/hashes": "^1.4.0",
"@open-rpc/client-js": "^1.8.1",
"@types/debug": "^4.1.12",

24
app/ydoc-shared/src/ast/__tests__/documentation.test.ts
View File

@ -1,3 +1,4 @@
import * as iter from 'enso-common/src/utilities/data/iter'
import { describe, expect, test } from 'vitest'
import { assert } from '../../util/assert'
import { MutableModule } from '../mutableModule'
@ -91,7 +92,7 @@ test('Creating comments: indented', () => {
expect(statement.module.root()?.code()).toBe(`main =\n ## ${docText}\n x = 1`)
})
describe('Markdown documentation', () => {
describe('Function documentation (Markdown)', () => {
const cases = [
{
source: '## My function',
@ -101,6 +102,10 @@ describe('Markdown documentation', () => {
source: '## My function\n\n Second paragraph',
markdown: 'My function\nSecond paragraph',
},
{
source: '## Trailing whitespace \n\n Second paragraph',
markdown: 'Trailing whitespace \nSecond paragraph',
},
{
source: '## My function\n\n\n Second paragraph after extra gap',
markdown: 'My function\n\nSecond paragraph after extra gap',
@ -141,14 +146,23 @@ describe('Markdown documentation', () => {
'the Enso syntax specification which requires line length not to exceed 100 characters.',
].join(' '), // TODO: This should be '\n ' when hard-wrapping is implemented.
},
{
source: '## Table below:\n | a | b |\n |---|---|',
markdown: 'Table below:\n| a | b |\n|---|---|',
},
{
source: '## Table below:\n\n | a | b |\n |---|---|',
markdown: 'Table below:\n\n| a | b |\n|---|---|',
},
]
test.each(cases)('Enso source comments to markdown', ({ source, markdown }) => {
test.each(cases)('Enso source comments to normalized markdown', ({ source, markdown }) => {
const moduleSource = `${source}\nmain =\n x = 1`
const topLevel = parseModule(moduleSource)
topLevel.module.setRoot(topLevel)
const main = [...topLevel.statements()][0]
const main = iter.first(topLevel.statements())
assert(main instanceof MutableFunctionDef)
expect(main.name.code()).toBe('main')
expect(main.mutableDocumentationMarkdown().toJSON()).toBe(markdown)
})
@ -156,7 +170,7 @@ describe('Markdown documentation', () => {
const functionCode = 'main =\n x = 1'
const topLevel = parseModule(functionCode)
topLevel.module.setRoot(topLevel)
const main = [...topLevel.statements()][0]
const main = iter.first(topLevel.statements())
assert(main instanceof MutableFunctionDef)
const markdownYText = main.mutableDocumentationMarkdown()
expect(markdownYText.toJSON()).toBe('')
@ -202,7 +216,7 @@ describe('Markdown documentation', () => {
const topLevel = parseModule(originalSourceWithDocComment)
expect(topLevel.code()).toBe(originalSourceWithDocComment)
const main = [...topLevel.statements()][0]
const main = iter.first(topLevel.statements())
assert(main instanceof MutableFunctionDef)
const markdownYText = main.mutableDocumentationMarkdown()
markdownYText.delete(0, markdownYText.length)

211
app/ydoc-shared/src/ast/documentation.ts
View File

@ -1,4 +1,5 @@
import { LINE_BOUNDARIES } from 'enso-common/src/utilities/data/string'
import { markdownParser } from './ensoMarkdown'
import { xxHash128 } from './ffi'
import type { ConcreteChild, RawConcreteChild } from './print'
import { ensureUnspaced, firstChild, preferUnspaced, unspaced } from './print'
@ -32,6 +33,8 @@ export function* docLineToConcrete(
for (const newline of docLine.newlines) yield preferUnspaced(newline)
}
// === Markdown ===
/**
* Render function documentation to concrete tokens. If the `markdown` content has the same value as when `docLine` was
* parsed (as indicated by `hash`), the `docLine` will be used (preserving concrete formatting). If it is different, the
@ -42,95 +45,161 @@ export function functionDocsToConcrete(
hash: string | undefined,
docLine: DeepReadonly<DocLine> | undefined,
indent: string | null,
): IterableIterator<RawConcreteChild> | undefined {
): Iterable<RawConcreteChild> | undefined {
return (
hash && docLine && xxHash128(markdown) === hash ? docLineToConcrete(docLine, indent)
: markdown ? yTextToTokens(markdown, (indent || '') + ' ')
: markdown ? markdownYTextToTokens(markdown, (indent || '') + ' ')
: undefined
)
}
function markdownYTextToTokens(yText: string, indent: string): Iterable<ConcreteChild<Token>> {
const tokensBuilder = new DocTokensBuilder(indent)
standardizeMarkdown(yText, tokensBuilder)
return tokensBuilder.build()
}
/**
* Given Enso documentation comment tokens, returns a model of their Markdown content. This model abstracts away details
* such as the locations of line breaks that are not paragraph breaks (e.g. lone newlines denoting hard-wrapping of the
* source code).
*/
export function abstractMarkdown(elements: undefined | TextToken<ConcreteRefs>[]) {
let markdown = ''
let newlines = 0
let readingTags = true
let elidedNewline = false
;(elements ?? []).forEach(({ token: { node } }, i) => {
if (node.tokenType_ === TokenType.Newline) {
if (readingTags || newlines > 0) {
markdown += '\n'
elidedNewline = false
} else {
elidedNewline = true
}
newlines += 1
} else {
let nodeCode = node.code()
if (i === 0) nodeCode = nodeCode.trimStart()
if (elidedNewline) markdown += ' '
markdown += nodeCode
newlines = 0
if (readingTags) {
if (!nodeCode.startsWith('ICON ')) {
readingTags = false
}
}
}
})
const { tags, rawMarkdown } = toRawMarkdown(elements)
const markdown = [...tags, normalizeMarkdown(rawMarkdown)].join('\n')
const hash = xxHash128(markdown)
return { markdown, hash }
}
// TODO: Paragraphs should be hard-wrapped to fit within the column limit, but this requires:
// - Recognizing block elements other than paragraphs; we must not split non-paragraph elements.
// - Recognizing inline elements; some cannot be split (e.g. links), while some can be broken into two (e.g. bold).
// If we break inline elements, we must also combine them when encountered during parsing.
const ENABLE_INCOMPLETE_WORD_WRAP_SUPPORT = false
function* yTextToTokens(yText: string, indent: string): IterableIterator<ConcreteChild<Token>> {
yield unspaced(Token.new('##', TokenType.TextStart))
const lines = yText.split(LINE_BOUNDARIES)
let printingTags = true
for (const [i, value] of lines.entries()) {
if (i) {
yield unspaced(Token.new('\n', TokenType.Newline))
if (value && !printingTags) yield unspaced(Token.new('\n', TokenType.Newline))
}
printingTags = printingTags && value.startsWith('ICON ')
let offset = 0
while (offset < value.length) {
if (offset !== 0) yield unspaced(Token.new('\n', TokenType.Newline))
let wrappedLineEnd = value.length
let printableOffset = offset
if (i !== 0) {
while (printableOffset < value.length && value[printableOffset] === ' ')
printableOffset += 1
function toRawMarkdown(elements: undefined | TextToken<ConcreteRefs>[]) {
const tags: string[] = []
let readingTags = true
let rawMarkdown = ''
;(elements ?? []).forEach(({ token: { node } }, i) => {
if (node.tokenType_ === TokenType.Newline) {
if (!readingTags) {
rawMarkdown += '\n'
}
if (ENABLE_INCOMPLETE_WORD_WRAP_SUPPORT && !printingTags) {
const ENSO_SOURCE_MAX_COLUMNS = 100
const MIN_DOC_COLUMNS = 40
const availableWidth = Math.max(
ENSO_SOURCE_MAX_COLUMNS - indent.length - (i === 0 && offset === 0 ? '## '.length : 0),
MIN_DOC_COLUMNS,
)
if (availableWidth < wrappedLineEnd - printableOffset) {
const wrapIndex = value.lastIndexOf(' ', printableOffset + availableWidth)
if (printableOffset < wrapIndex) {
wrappedLineEnd = wrapIndex
}
} else {
let nodeCode = node.code()
if (i === 0) nodeCode = nodeCode.trimStart()
if (readingTags) {
if (nodeCode.startsWith('ICON ')) {
tags.push(nodeCode)
} else {
readingTags = false
}
}
while (printableOffset < value.length && value[printableOffset] === ' ') printableOffset += 1
const whitespace = i === 0 && offset === 0 ? ' ' : indent
const wrappedLine = value.substring(printableOffset, wrappedLineEnd)
yield { whitespace, node: Token.new(wrappedLine, TokenType.TextSection) }
offset = wrappedLineEnd
if (!readingTags) {
rawMarkdown += nodeCode
}
}
}
yield unspaced(Token.new('\n', TokenType.Newline))
})
return { tags, rawMarkdown }
}
/**
* Convert the Markdown input to a format with rendered-style linebreaks: Hard-wrapped lines within a paragraph will be
* joined, and only a single linebreak character is used to separate paragraphs.
*/
function normalizeMarkdown(rawMarkdown: string): string {
let normalized = ''
let prevTo = 0
let prevName: string | undefined = undefined
const cursor = markdownParser.parse(rawMarkdown).cursor()
cursor.firstChild()
do {
if (prevTo < cursor.from) {
const textBetween = rawMarkdown.slice(prevTo, cursor.from)
normalized +=
cursor.name === 'Paragraph' && prevName !== 'Table' ? textBetween.slice(0, -1) : textBetween
}
const text = rawMarkdown.slice(cursor.from, cursor.to)
normalized += cursor.name === 'Paragraph' ? text.replaceAll(/ *\n */g, ' ') : text
prevTo = cursor.to
prevName = cursor.name
} while (cursor.nextSibling())
return normalized
}
/**
* Convert from "normalized" Markdown to the on-disk representation, with paragraphs hard-wrapped and separated by blank
* lines.
*/
function standardizeMarkdown(normalizedMarkdown: string, textConsumer: TextConsumer) {
let prevTo = 0
let prevName: string | undefined = undefined
let printingTags = true
const cursor = markdownParser.parse(normalizedMarkdown).cursor()
cursor.firstChild()
do {
if (prevTo < cursor.from) {
const betweenText = normalizedMarkdown.slice(prevTo, cursor.from)
for (const _match of betweenText.matchAll(LINE_BOUNDARIES)) {
textConsumer.newline()
}
if (cursor.name === 'Paragraph' && prevName !== 'Table') {
textConsumer.newline()
}
}
const lines = normalizedMarkdown.slice(cursor.from, cursor.to).split(LINE_BOUNDARIES)
if (cursor.name === 'Paragraph') {
let printingNonTags = false
lines.forEach((line, i) => {
if (printingTags) {
if (cursor.name === 'Paragraph' && line.startsWith('ICON ')) {
textConsumer.text(line)
} else {
printingTags = false
}
}
if (!printingTags) {
if (i > 0) {
textConsumer.newline()
if (printingNonTags) textConsumer.newline()
}
textConsumer.wrapText(line)
printingNonTags = true
}
})
} else {
lines.forEach((line, i) => {
if (i > 0) textConsumer.newline()
textConsumer.text(line)
})
printingTags = false
}
prevTo = cursor.to
prevName = cursor.name
} while (cursor.nextSibling())
}
interface TextConsumer {
text: (text: string) => void
wrapText: (text: string) => void
newline: () => void
}
class DocTokensBuilder implements TextConsumer {
private readonly tokens: ConcreteChild<Token>[] = [unspaced(Token.new('##', TokenType.TextStart))]
constructor(private readonly indent: string) {}
text(text: string): void {
const whitespace = this.tokens.length === 1 ? ' ' : this.indent
this.tokens.push({ whitespace, node: Token.new(text, TokenType.TextSection) })
}
wrapText(text: string): void {
this.text(text)
}
newline(): void {
this.tokens.push(unspaced(Token.new('\n', TokenType.Newline)))
}
build(): ConcreteChild<Token>[] {
this.newline()
return this.tokens
}
}

54
app/gui/src/project-view/components/MarkdownEditor/markdown/parse.ts → app/ydoc-shared/src/ast/ensoMarkdown.ts
View File

@ -1,6 +1,4 @@
import { markdown as baseMarkdown, markdownLanguage } from '@codemirror/lang-markdown'
import type { Extension } from '@codemirror/state'
import type { Tree } from '@lezer/common'
import { TreeCursor } from '@lezer/common'
import type {
BlockContext,
BlockParser,
@ -12,31 +10,11 @@ import type {
MarkdownParser,
NodeSpec,
} from '@lezer/markdown'
import { Element } from '@lezer/markdown'
import { parser as baseParser, Element, Emoji, GFM, Subscript, Superscript } from '@lezer/markdown'
import { assertDefined } from 'ydoc-shared/util/assert'
/**
* Enso Markdown extension. Differences from CodeMirror's base Markdown extension:
* - It defines the flavor of Markdown supported in Enso documentation. Currently, this is mostly CommonMark except we
* don't support setext headings. Planned features include support for some GFM extensions.
* - Many of the parsers differ from the `@lezer/markdown` parsers in their treatment of whitespace, in order to support
* a rendering mode where markup (and some associated spacing) is hidden.
*/
export function markdown(): Extension {
return baseMarkdown({
base: markdownLanguage,
extensions: [
{
parseBlock: [headerParser, bulletList, orderedList, blockquoteParser, disableSetextHeading],
parseInline: [linkParser, imageParser, linkEndParser],
defineNodes: [blockquoteNode],
},
],
})
}
function getType({ parser }: { parser: MarkdownParser }, name: string) {
const ty = parser.nodeSet.types.find((ty) => ty.name === name)
const ty = parser.nodeSet.types.find(ty => ty.name === name)
assertDefined(ty)
return ty.id
}
@ -424,12 +402,12 @@ export interface DebugTree {
// noinspection JSUnusedGlobalSymbols
/** @returns A debug representation of the provided {@link Tree} */
export function debugTree(tree: Tree): DebugTree {
export function debugTree(tree: { cursor: () => TreeCursor }): DebugTree {
const cursor = tree.cursor()
let current: DebugTree[] = []
const stack: DebugTree[][] = []
cursor.iterate(
(node) => {
node => {
const children: DebugTree[] = []
current.push({
name: node.name,
@ -463,3 +441,25 @@ function isAtxHeading(line: Line) {
function isSpace(ch: number) {
return ch == 32 || ch == 9 || ch == 10 || ch == 13
}
const ensoMarkdownLanguageExtension = {
parseBlock: [headerParser, bulletList, orderedList, blockquoteParser, disableSetextHeading],
parseInline: [linkParser, imageParser, linkEndParser],
defineNodes: [blockquoteNode],
}
/**
* Lezer (CodeMirror) parser for the Enso documentation Markdown dialect.
* Differences from CodeMirror's base Markdown language:
* - It defines the flavor of Markdown supported in Enso documentation. Currently, this is mostly CommonMark except we
* don't support setext headings. Planned features include support for some GFM extensions.
* - Many of the parsers differ from the `@lezer/markdown` parsers in their treatment of whitespace, in order to support
* a rendering mode where markup (and some associated spacing) is hidden.
*/
export const markdownParser: MarkdownParser = baseParser.configure([
GFM,
Subscript,
Superscript,
Emoji,
ensoMarkdownLanguageExtension,
])

0
app/gui/src/project-view/components/MarkdownEditor/markdown/lezer.d.ts → app/ydoc-shared/src/ast/lezerMarkdown.d.ts vendored
View File

35
app/ydoc-shared/src/ast/parse.ts
View File

@ -534,3 +534,38 @@ export function setExternalIds(edit: MutableModule, spans: SpanMap, ids: IdMap):
}
return astsMatched
}
/**
* Determines the context of `ast`: module root, body block, statement, or expression; parses the given code in the same
* context.
*/
export function parseInSameContext(
module: MutableModule,
code: string,
ast: Ast,
): { root: Owned; spans: SpanMap; toRaw: Map<AstId, RawAst.Tree> } {
const rawParsed = rawParseInContext(code, getParseContext(ast))
return abstract(module, rawParsed, code)
}
type ParseContext = 'module' | 'block' | 'expression' | 'statement'
function getParseContext(ast: Ast): ParseContext {
const astModuleRoot = ast.module.root()
if (ast instanceof BodyBlock) return astModuleRoot && ast.is(astModuleRoot) ? 'module' : 'block'
return ast.isExpression() ? 'expression' : 'statement'
}
function rawParseInContext(code: string, context: ParseContext): RawAst.Tree {
if (context === 'module') return rawParseModule(code)
const block = rawParseBlock(code)
if (context === 'block') return block
const statement = iter.tryGetSoleValue(block.statements)?.expression
if (!statement) return block
if (context === 'statement') return statement
if (context === 'expression')
return statement.type === RawAst.Tree.Type.ExpressionStatement ?
statement.expression
: statement
return context satisfies never
}

61
app/ydoc-shared/src/ast/syncToCode.ts
View File

@ -1,30 +1,35 @@
import * as iter from 'enso-common/src/utilities/data/iter'
import * as map from 'lib0/map'
import { assert, assertDefined } from '../util/assert'
import type { SourceRangeEdit, SpanTree } from '../util/data/text'
import {
type SourceRangeEdit,
type SpanTree,
applyTextEdits,
applyTextEditsToSpans,
enclosingSpans,
textChangeToEdits,
trimEnd,
} from '../util/data/text'
import type { SourceRange, SourceRangeKey } from '../yjsModel'
import { rangeLength, sourceRangeFromKey, sourceRangeKey } from '../yjsModel'
import {
type SourceRange,
type SourceRangeKey,
rangeLength,
sourceRangeFromKey,
sourceRangeKey,
} from '../yjsModel'
import { xxHash128 } from './ffi'
import * as RawAst from './generated/ast'
import type { NodeKey, NodeSpanMap } from './idMap'
import { newExternalId } from './idMap'
import { type NodeKey, type NodeSpanMap, newExternalId } from './idMap'
import type { Module, MutableModule } from './mutableModule'
import { abstract, rawParseBlock, rawParseModule } from './parse'
import { parseInSameContext } from './parse'
import { printWithSpans } from './print'
import { isTokenId } from './token'
import type { AstId, MutableAst, Owned } from './tree'
import {
Assignment,
Ast,
type AstId,
MutableAssignment,
MutableBodyBlock,
type MutableAst,
type Owned,
rewriteRefs,
syncFields,
syncNodeMetadata,
@ -32,7 +37,6 @@ import {
/**
* Recursion helper for {@link syntaxHash}.
* @internal
*/
function hashSubtreeSyntax(ast: Ast, hashesOut: Map<SyntaxHash, Ast[]>): SyntaxHash {
let content = ''
@ -53,6 +57,7 @@ function hashSubtreeSyntax(ast: Ast, hashesOut: Map<SyntaxHash, Ast[]>): SyntaxH
declare const brandHash: unique symbol
/** See {@link syntaxHash}. */
type SyntaxHash = string & { [brandHash]: never }
/** Applies the syntax-data hashing function to the input, and brands the result as a `SyntaxHash`. */
function hashString(input: string): SyntaxHash {
return xxHash128(input) as SyntaxHash
@ -170,32 +175,18 @@ export function applyTextEditsToAst(
) {
const printed = printWithSpans(ast)
const code = applyTextEdits(printed.code, textEdits)
const astModuleRoot = ast.module.root()
const rawParsedBlock =
ast instanceof MutableBodyBlock && astModuleRoot && ast.is(astModuleRoot) ?
rawParseModule(code)
: rawParseBlock(code)
const rawParsedStatement =
ast instanceof MutableBodyBlock ? undefined : (
iter.tryGetSoleValue(rawParsedBlock.statements)?.expression
ast.module.transact(() => {
const parsed = parseInSameContext(ast.module, code, ast)
const toSync = calculateCorrespondence(
ast,
printed.info.nodes,
parsed.root,
parsed.spans.nodes,
textEdits,
code,
)
const rawParsedExpression =
ast.isExpression() ?
rawParsedStatement?.type === RawAst.Tree.Type.ExpressionStatement ?
rawParsedStatement.expression
: undefined
: undefined
const rawParsed = rawParsedExpression ?? rawParsedStatement ?? rawParsedBlock
const parsed = abstract(ast.module, rawParsed, code)
const toSync = calculateCorrespondence(
ast,
printed.info.nodes,
parsed.root,
parsed.spans.nodes,
textEdits,
code,
)
syncTree(ast, parsed.root, toSync, ast.module, metadataSource)
syncTree(ast, parsed.root, toSync, ast.module, metadataSource)
})
}
/** Replace `target` with `newContent`, reusing nodes according to the correspondence in `toSync`. */

7
app/ydoc-shared/src/ast/tree.ts
View File

@ -565,8 +565,11 @@ export function syncFields(ast1: MutableAst, ast2: Ast, f: (id: AstId) => AstId
}
function syncYText(target: Y.Text, source: Y.Text) {
target.delete(0, target.length)
target.insert(0, source.toJSON())
const sourceString = source.toJSON()
if (target.toJSON() !== sourceString) {
target.delete(0, target.length)
target.insert(0, sourceString)
}
}
/** TODO: Add docs */

26
pnpm-lock.yaml
View File

@ -172,9 +172,6 @@ importers:
'@lezer/highlight':
specifier: ^1.1.6
version: 1.2.0
'@lezer/markdown':
specifier: ^1.3.1
version: 1.3.1
'@monaco-editor/react':
specifier: 4.6.0
version: 4.6.0(monaco-editor@0.48.0)(react-dom@18.3.1(react@18.3.1))(react@18.3.1)
@ -343,9 +340,6 @@ importers:
vue-component-type-helpers:
specifier: ^2.0.29
version: 2.0.29
y-codemirror.next:
specifier: ^0.3.2
version: 0.3.5(@codemirror/state@6.4.1)(@codemirror/view@6.28.3)(yjs@13.6.18)
y-protocols:
specifier: ^1.0.5
version: 1.0.6(yjs@13.6.18)
@ -755,6 +749,12 @@ importers:
app/ydoc-shared:
dependencies:
'@lezer/common':
specifier: ^1.1.0
version: 1.2.1
'@lezer/markdown':
specifier: ^1.3.1
version: 1.3.1
'@noble/hashes':
specifier: ^1.4.0
version: 1.4.0
@ -7311,13 +7311,6 @@ packages:
resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==}
engines: {node: '>=0.4'}
y-codemirror.next@0.3.5:
resolution: {integrity: sha512-VluNu3e5HfEXybnypnsGwKAj+fKLd4iAnR7JuX1Sfyydmn1jCBS5wwEL/uS04Ch2ib0DnMAOF6ZRR/8kK3wyGw==}
peerDependencies:
'@codemirror/state': ^6.0.0
'@codemirror/view': ^6.0.0
yjs: ^13.5.6
y-leveldb@0.1.2:
resolution: {integrity: sha512-6ulEn5AXfXJYi89rXPEg2mMHAyyw8+ZfeMMdOtBbV8FJpQ1NOrcgi6DTAcXof0dap84NjHPT2+9d0rb6cFsjEg==}
peerDependencies:
@ -15325,13 +15318,6 @@ snapshots:
xtend@4.0.2: {}
y-codemirror.next@0.3.5(@codemirror/state@6.4.1)(@codemirror/view@6.28.3)(yjs@13.6.18):
dependencies:
'@codemirror/state': 6.4.1
'@codemirror/view': 6.28.3
lib0: 0.2.94
yjs: 13.6.18
y-leveldb@0.1.2(yjs@13.6.18):
dependencies:
level: 6.0.1