Ast.Vector (#9328)

Add `Vector` AST type, corresponding to the `RawAst.Tree.Array` type (name `Array` not used for obvious reasons).

This is the first step of #5138.

### Important Notes

- Switched some string-based vector construction to `Vector.new`, improving type-safety.
- The `Ast` changes are covered by the round-trip tests; the use-site changes have been tested manually.

app/gui2/shared/ast/parse.ts

@@ -57,6 +57,7 @@ import {
   PropertyAccess,
   TextLiteral,
   UnaryOprApp,
+  Vector,
   Wildcard,
 } from './tree'
 
@@ -290,6 +291,20 @@ class Abstractor {
         node = Import.concrete(this.module, polyglot, from, import_, all, as, hiding)
         break
       }
+      case RawAst.Tree.Type.Array: {
+        const left = this.abstractToken(tree.left)
+        const elements = []
+        if (tree.first) elements.push({ value: this.abstractTree(tree.first) })
+        for (const rawElement of tree.rest) {
+          elements.push({
+            delimiter: this.abstractToken(rawElement.operator),
+            value: rawElement.body && this.abstractTree(rawElement.body),
+          })
+        }
+        const right = this.abstractToken(tree.right)
+        node = Vector.concrete(this.module, left, elements, right)
+        break
+      }
       default: {
         node = Generic.concrete(this.module, this.abstractChildren(tree))
       }

app/gui2/shared/ast/tree.ts

@@ -360,6 +360,7 @@ type StructuralField<T extends TreeRefs = RawRefs> =
   | NameSpecification<T>
   | TextElement<T>
   | ArgumentDefinition<T>
+  | VectorElement<T>
 
 /** Type whose fields are all suitable for storage as `Ast` fields. */
 interface FieldObject<T extends TreeRefs> {
@@ -470,6 +471,10 @@ function mapRefs<T extends TreeRefs, U extends TreeRefs>(
   field: ArgumentDefinition<T>,
   f: MapRef<T, U>,
 ): ArgumentDefinition<U>
+function mapRefs<T extends TreeRefs, U extends TreeRefs>(
+  field: VectorElement<T>,
+  f: MapRef<T, U>,
+): VectorElement<U>
 function mapRefs<T extends TreeRefs, U extends TreeRefs>(
   field: FieldData<T>,
   f: MapRef<T, U>,
@@ -646,6 +651,9 @@ function ensureUnspaced<T>(child: NodeChild<T>, verbatim: boolean | undefined):
 function preferUnspaced<T>(child: NodeChild<T>): NodeChild<T> {
   return child.whitespace === undefined ? { whitespace: '', ...child } : child
 }
+function preferSpaced<T>(child: NodeChild<T>): NodeChild<T> {
+  return child.whitespace === undefined ? { whitespace: ' ', ...child } : child
+}
 export class MutableApp extends App implements MutableAst {
   declare readonly module: MutableModule
   declare readonly fields: FixedMap<AstFields & AppFields>
@@ -1335,7 +1343,7 @@ export class TextLiteral extends Ast {
     if (!(parsed instanceof MutableTextLiteral)) {
       console.error(`Failed to escape string for interpolated text`, rawText, escaped, parsed)
       const safeText = rawText.replaceAll(/[^-+A-Za-z0-9_. ]/g, '')
-      return this.new(safeText, module)
+      return TextLiteral.new(safeText, module)
     }
     return parsed
   }
@@ -2137,6 +2145,116 @@ export class MutableWildcard extends Wildcard implements MutableAst {
 export interface MutableWildcard extends Wildcard, MutableAst {}
 applyMixins(MutableWildcard, [MutableAst])
 
+type AbstractVectorElement<T extends TreeRefs> = {
+  delimiter?: T['token']
+  value: T['ast'] | undefined
+}
+function delimitVectorElement(element: AbstractVectorElement<OwnedRefs>): VectorElement<OwnedRefs> {
+  return {
+    ...element,
+    delimiter: element.delimiter ?? unspaced(Token.new(',', RawAst.Token.Type.Operator)),
+  }
+}
+type VectorElement<T extends TreeRefs> = { delimiter: T['token']; value: T['ast'] | undefined }
+interface VectorFields {
+  open: NodeChild<Token>
+  elements: VectorElement<RawRefs>[]
+  close: NodeChild<Token>
+}
+export class Vector extends Ast {
+  declare fields: FixedMapView<AstFields & VectorFields>
+  constructor(module: Module, fields: FixedMapView<AstFields & VectorFields>) {
+    super(module, fields)
+  }
+
+  static concrete(
+    module: MutableModule,
+    open: NodeChild<Token> | undefined,
+    elements: AbstractVectorElement<OwnedRefs>[],
+    close: NodeChild<Token> | undefined,
+  ) {
+    const base = module.baseObject('Vector')
+    const id_ = base.get('id')
+    const fields = composeFieldData(base, {
+      open: open ?? unspaced(Token.new('[', RawAst.Token.Type.OpenSymbol)),
+      elements: elements.map(delimitVectorElement).map((e) => mapRefs(e, ownedToRaw(module, id_))),
+      close: close ?? unspaced(Token.new(']', RawAst.Token.Type.CloseSymbol)),
+    })
+    return asOwned(new MutableVector(module, fields))
+  }
+
+  static new(module: MutableModule, elements: Owned[]) {
+    return this.concrete(
+      module,
+      undefined,
+      elements.map((value) => ({ value: autospaced(value) })),
+      undefined,
+    )
+  }
+
+  static tryBuild<T>(
+    inputs: Iterable<T>,
+    elementBuilder: (input: T, module: MutableModule) => Owned,
+    edit?: MutableModule,
+  ): Owned<MutableVector>
+  static tryBuild<T>(
+    inputs: Iterable<T>,
+    elementBuilder: (input: T, module: MutableModule) => Owned | undefined,
+    edit?: MutableModule,
+  ): Owned<MutableVector> | undefined
+  static tryBuild<T>(
+    inputs: Iterable<T>,
+    valueBuilder: (input: T, module: MutableModule) => Owned | undefined,
+    edit?: MutableModule,
+  ): Owned<MutableVector> | undefined {
+    const module = edit ?? MutableModule.Transient()
+    const elements = new Array<AbstractVectorElement<OwnedRefs>>()
+    for (const input of inputs) {
+      const value = valueBuilder(input, module)
+      if (!value) return
+      elements.push({ value: autospaced(value) })
+    }
+    return Vector.concrete(module, undefined, elements, undefined)
+  }
+
+  static build<T>(
+    inputs: Iterable<T>,
+    elementBuilder: (input: T, module: MutableModule) => Owned,
+    edit?: MutableModule,
+  ): Owned<MutableVector> {
+    return Vector.tryBuild(inputs, elementBuilder, edit)
+  }
+
+  *concreteChildren(_verbatim?: boolean): IterableIterator<RawNodeChild> {
+    const { open, elements, close } = getAll(this.fields)
+    yield unspaced(open.node)
+    let isFirst = true
+    for (const { delimiter, value } of elements) {
+      if (isFirst && value) {
+        yield preferUnspaced(value)
+      } else {
+        yield delimiter
+        if (value) yield preferSpaced(value)
+      }
+      isFirst = false
+    }
+    yield preferUnspaced(close)
+  }
+
+  *values(): IterableIterator<Ast> {
+    for (const element of this.fields.get('elements'))
+      if (element.value) yield this.module.get(element.value.node)
+  }
+}
+export class MutableVector extends Vector implements MutableAst {
+  declare readonly module: MutableModule
+  declare readonly fields: FixedMap<AstFields & VectorFields>
+}
+export interface MutableVector extends Vector, MutableAst {
+  values(): IterableIterator<MutableAst>
+}
+applyMixins(MutableVector, [MutableAst])
+
 export type Mutable<T extends Ast = Ast> =
   T extends App ? MutableApp
   : T extends Assignment ? MutableAssignment
@@ -2154,6 +2272,7 @@ export type Mutable<T extends Ast = Ast> =
   : T extends PropertyAccess ? MutablePropertyAccess
   : T extends TextLiteral ? MutableTextLiteral
   : T extends UnaryOprApp ? MutableUnaryOprApp
+  : T extends Vector ? MutableVector
   : T extends Wildcard ? MutableWildcard
   : MutableAst
 
@@ -2163,36 +2282,38 @@ export function materializeMutable(module: MutableModule, fields: FixedMap<AstFi
   switch (type) {
     case 'App':
       return new MutableApp(module, fieldsForType)
-    case 'UnaryOprApp':
-      return new MutableUnaryOprApp(module, fieldsForType)
-    case 'NegationApp':
-      return new MutableNegationApp(module, fieldsForType)
-    case 'OprApp':
-      return new MutableOprApp(module, fieldsForType)
-    case 'PropertyAccess':
-      return new MutablePropertyAccess(module, fieldsForType)
-    case 'Generic':
-      return new MutableGeneric(module, fieldsForType)
-    case 'Import':
-      return new MutableImport(module, fieldsForType)
-    case 'TextLiteral':
-      return new MutableTextLiteral(module, fieldsForType)
-    case 'Documented':
-      return new MutableDocumented(module, fieldsForType)
-    case 'Invalid':
-      return new MutableInvalid(module, fieldsForType)
-    case 'Group':
-      return new MutableGroup(module, fieldsForType)
-    case 'NumericLiteral':
-      return new MutableNumericLiteral(module, fieldsForType)
-    case 'Function':
-      return new MutableFunction(module, fieldsForType)
     case 'Assignment':
       return new MutableAssignment(module, fieldsForType)
     case 'BodyBlock':
       return new MutableBodyBlock(module, fieldsForType)
+    case 'Documented':
+      return new MutableDocumented(module, fieldsForType)
+    case 'Function':
+      return new MutableFunction(module, fieldsForType)
+    case 'Generic':
+      return new MutableGeneric(module, fieldsForType)
+    case 'Group':
+      return new MutableGroup(module, fieldsForType)
     case 'Ident':
       return new MutableIdent(module, fieldsForType)
+    case 'Import':
+      return new MutableImport(module, fieldsForType)
+    case 'Invalid':
+      return new MutableInvalid(module, fieldsForType)
+    case 'NegationApp':
+      return new MutableNegationApp(module, fieldsForType)
+    case 'NumericLiteral':
+      return new MutableNumericLiteral(module, fieldsForType)
+    case 'OprApp':
+      return new MutableOprApp(module, fieldsForType)
+    case 'PropertyAccess':
+      return new MutablePropertyAccess(module, fieldsForType)
+    case 'TextLiteral':
+      return new MutableTextLiteral(module, fieldsForType)
+    case 'UnaryOprApp':
+      return new MutableUnaryOprApp(module, fieldsForType)
+    case 'Vector':
+      return new MutableVector(module, fieldsForType)
     case 'Wildcard':
       return new MutableWildcard(module, fieldsForType)
   }
@@ -2205,36 +2326,38 @@ export function materialize(module: Module, fields: FixedMapView<AstFields>): As
   switch (type) {
     case 'App':
       return new App(module, fields_)
-    case 'UnaryOprApp':
-      return new UnaryOprApp(module, fields_)
-    case 'NegationApp':
-      return new NegationApp(module, fields_)
-    case 'OprApp':
-      return new OprApp(module, fields_)
-    case 'PropertyAccess':
-      return new PropertyAccess(module, fields_)
-    case 'Generic':
-      return new Generic(module, fields_)
-    case 'Import':
-      return new Import(module, fields_)
-    case 'TextLiteral':
-      return new TextLiteral(module, fields_)
-    case 'Documented':
-      return new Documented(module, fields_)
-    case 'Invalid':
-      return new Invalid(module, fields_)
-    case 'Group':
-      return new Group(module, fields_)
-    case 'NumericLiteral':
-      return new NumericLiteral(module, fields_)
-    case 'Function':
-      return new Function(module, fields_)
     case 'Assignment':
       return new Assignment(module, fields_)
     case 'BodyBlock':
       return new BodyBlock(module, fields_)
+    case 'Documented':
+      return new Documented(module, fields_)
+    case 'Function':
+      return new Function(module, fields_)
+    case 'Generic':
+      return new Generic(module, fields_)
+    case 'Group':
+      return new Group(module, fields_)
     case 'Ident':
       return new Ident(module, fields_)
+    case 'Import':
+      return new Import(module, fields_)
+    case 'Invalid':
+      return new Invalid(module, fields_)
+    case 'NegationApp':
+      return new NegationApp(module, fields_)
+    case 'NumericLiteral':
+      return new NumericLiteral(module, fields_)
+    case 'OprApp':
+      return new OprApp(module, fields_)
+    case 'PropertyAccess':
+      return new PropertyAccess(module, fields_)
+    case 'TextLiteral':
+      return new TextLiteral(module, fields_)
+    case 'UnaryOprApp':
+      return new UnaryOprApp(module, fields_)
+    case 'Vector':
+      return new Vector(module, fields_)
     case 'Wildcard':
       return new Wildcard(module, fields_)
   }

app/gui2/src/components/GraphEditor/GraphEdges.vue

@@ -75,7 +75,6 @@ function disconnectEdge(target: PortId) {
 function createEdge(source: AstId, target: PortId) {
   const ident = graph.db.getOutputPortIdentifier(source)
   if (ident == null) return
-  const identAst = Ast.parse(ident)
 
   const sourceNode = graph.db.getPatternExpressionNodeId(source)
   const targetNode = graph.getPortNodeId(target)
@@ -89,6 +88,7 @@ function createEdge(source: AstId, target: PortId) {
     // Creating this edge would create a circular dependency. Prevent that and display error.
     toast.error('Could not connect due to circular dependency.')
   } else {
+    const identAst = Ast.parse(ident, edit)
     if (!graph.updatePortValue(edit, target, identAst)) {
       if (isAstId(target)) {
         console.warn(`Failed to connect edge to port ${target}, falling back to direct edit.`)

app/gui2/src/components/GraphEditor/widgets/WidgetFunction.vue

@@ -95,12 +95,6 @@ const innerInput = computed(() => {
   }
 })
 
-const escapeString = (str: string): string => {
-  const escaped = str.replaceAll(/([\\'])/g, '\\$1')
-  return `'${escaped}'`
-}
-const makeArgsList = (args: string[]) => '[' + args.map(escapeString).join(', ') + ']'
-
 const selfArgumentExternalId = computed<Opt<ExternalId>>(() => {
   const analyzed = interpretCall(props.input.value, true)
   if (analyzed.kind === 'infix') {
@@ -136,7 +130,10 @@ const visualizationConfig = computed<Opt<NodeVisualizationConfiguration>>(() =>
       definedOnType: 'Standard.Visualization.Widgets',
       name: 'get_widget_json',
     },
-    positionalArgumentsExpressions: [`.${name}`, makeArgsList(args)],
+    positionalArgumentsExpressions: [
+      `.${name}`,
+      Ast.Vector.build(args, Ast.TextLiteral.new).code(),
+    ],
   }
 })
 

app/gui2/src/components/GraphEditor/widgets/WidgetText.vue

@@ -13,9 +13,8 @@ const graph = useGraphStore()
 const inputTextLiteral = computed((): Ast.TextLiteral | undefined => {
   if (props.input.value instanceof Ast.TextLiteral) return props.input.value
   const valueStr = WidgetInput.valueRepr(props.input)
-  const parsed = valueStr != null ? Ast.parse(valueStr) : undefined
-  if (parsed instanceof Ast.TextLiteral) return parsed
-  return undefined
+  if (valueStr == null) return undefined
+  return Ast.TextLiteral.tryParse(valueStr)
 })
 
 function makeNewLiteral(value: string) {

app/gui2/src/components/GraphEditor/widgets/WidgetVector.vue

@@ -25,16 +25,19 @@ const defaultItem = computed(() => {
 
 const value = computed({
   get() {
-    if (!(props.input.value instanceof Ast.Ast)) return []
-    return Array.from(props.input.value.children()).filter(
-      (child): child is Ast.Ast => child instanceof Ast.Ast,
-    )
+    return props.input.value instanceof Ast.Vector ? [...props.input.value.values()] : []
   },
   set(value) {
-    // TODO[ao]: here we re-create AST. It would be better to reuse existing AST nodes.
-    const newCode = `[${value.map((item) => item.code()).join(', ')}]`
+    // This doesn't preserve AST identities, because the values are not `Ast.Owned`.
+    // Getting/setting an Array is incompatible with ideal synchronization anyway;
+    // `ListWidget` needs to operate on the `Ast.Vector` for edits to be merged as `Y.Array` operations.
+    const tempModule = MutableModule.Transient()
+    const newAst = Ast.Vector.new(
+      tempModule,
+      value.map((element) => tempModule.copy(element)),
+    )
     props.onUpdate({
-      portUpdate: { value: newCode, origin: props.input.portId },
+      portUpdate: { value: newAst, origin: props.input.portId },
     })
   },
 })
@@ -45,16 +48,11 @@ const navigator = injectGraphNavigator(true)
 <script lang="ts">
 export const widgetDefinition = defineWidget(WidgetInput.isAstOrPlaceholder, {
   priority: 500,
-  score: (props) => {
-    if (props.input.dynamicConfig?.kind === 'Vector_Editor') return Score.Perfect
-    else if (props.input.expectedType?.startsWith('Standard.Base.Data.Vector.Vector'))
-      return Score.Good
-    else if (props.input.value instanceof Ast.Ast) {
-      return props.input.value.children().next().value.code() === '[' ?
-          Score.Perfect
-        : Score.Mismatch
-    } else return Score.Mismatch
-  },
+  score: (props) =>
+    props.input.dynamicConfig?.kind === 'Vector_Editor' ? Score.Perfect
+    : props.input.value instanceof Ast.Vector ? Score.Perfect
+    : props.input.expectedType?.startsWith('Standard.Base.Data.Vector.Vector') ? Score.Good
+    : Score.Mismatch,
 })
 </script>
 

app/gui2/src/components/visualizations/ScatterplotVisualization.vue

@@ -2,6 +2,8 @@
 import SvgIcon from '@/components/SvgIcon.vue'
 import { useEvent } from '@/composables/events'
 import { useVisualizationConfig } from '@/providers/visualizationConfig'
+import { Ast } from '@/util/ast'
+import { tryNumberToEnso } from '@/util/ast/abstract'
 import { getTextWidthBySizeAndFamily } from '@/util/measurement'
 import { VisualizationContainer, defineKeybinds } from '@/util/visualizationBuiltins'
 import { computed, ref, watch, watchEffect, watchPostEffect } from 'vue'
@@ -232,11 +234,13 @@ const yLabelLeft = computed(
 const yLabelTop = computed(() => -margin.value.left + 15)
 
 watchEffect(() => {
+  const boundsExpression =
+    bounds.value != null ? Ast.Vector.tryBuild(bounds.value, tryNumberToEnso) : undefined
   emit(
     'update:preprocessor',
     'Standard.Visualization.Scatter_Plot',
     'process_to_json_text',
-    bounds.value == null ? 'Nothing' : '[' + bounds.value.join(',') + ']',
+    boundsExpression?.code() ?? 'Nothing',
     limit.value.toString(),
   )
 })

app/gui2/src/util/ast/__tests__/abstract.test.ts

@@ -275,6 +275,9 @@ const cases = [
   '{x, y}',
   '[ x , y , z ]',
   '[x, y, z]',
+  '[x ,y ,z]',
+  '[,,,,,]',
+  '[]',
   'x + y * z',
   'x * y + z',
   ["'''", ' `splice` at start'].join('\n'),

app/gui2/src/util/ast/abstract.ts

@@ -1,4 +1,3 @@
-import { parseEnso } from '@/util/ast'
 import { normalizeQualifiedName, qnFromSegments } from '@/util/qualifiedName'
 import type {
   AstId,
@@ -17,10 +16,10 @@ import {
   Ident,
   MutableBodyBlock,
   MutableModule,
+  NumericLiteral,
   OprApp,
   PropertyAccess,
   Token,
-  abstract,
   isTokenId,
   print,
 } from 'shared/ast'
@@ -28,13 +27,9 @@ export * from 'shared/ast'
 
 export function deserialize(serialized: string): Owned {
   const parsed: SerializedPrintedSource = JSON.parse(serialized)
-  const module = MutableModule.Transient()
-  const tree = parseEnso(parsed.code)
-  const ast = abstract(module, tree, parsed.code)
-  // const nodes = new Map(unsafeEntries(parsed.info.nodes))
-  // const tokens = new Map(unsafeEntries(parsed.info.tokens))
-  // TODO: ast <- nodes,tokens
-  return ast.root
+  // Not implemented: restoring serialized external IDs. This is not the best approach anyway;
+  // Y.Js can't merge edits to objects when they're being serialized and deserialized.
+  return Ast.parse(parsed.code)
 }
 
 interface SerializedInfoMap {
@@ -197,6 +192,19 @@ export function substituteQualifiedName(
   }
 }
 
+/** Try to convert the number to an Enso value.
+ *
+ *  Returns `undefined` if the input is not a real number. NOTE: The current implementation doesn't support numbers that
+ *  JS prints in scientific notation.
+ */
+export function tryNumberToEnso(value: number, module: MutableModule) {
+  if (!Number.isFinite(value)) return
+  const literal = NumericLiteral.tryParse(value.toString(), module)
+  if (!literal)
+    console.warn(`Not implemented: Converting scientific-notation number to Enso value`, value)
+  return literal
+}
+
 declare const tokenKey: unique symbol
 declare module '@/providers/widgetRegistry' {
   export interface WidgetInputTypes {