enso/app/gui2/shared/ast/token.ts
Kaz Wesley 343a644051
Syntactic synchronization, automatic parentheses, metadata in Ast (#8893)
- Synchronize Y.Js clients by AST (implements #8237).
- Before committing an edit, insert any parentheses-nodes needed for the concrete syntax to reflect tree structure (fixes #8884).
- Move `externalId` and all node metadata into a Y.Map owned by each `Ast`. This allows including metadata changes in an edit, enables Y.Js merging of changes to different metadata fields, and will enable the use of Y.Js objects in metadata. (Implements #8804.)

### Important Notes

- Metadata is now set and retrieved through accessors on the `Ast` objects.
- Since some metadata edits need to take effect in real time (e.g. node dragging), new lower-overhead APIs (`commitDirect`, `skipTreeRepair`) are provided for careful use in certain cases.
- The client is now bundled as ESM.
- The build script cleans up git-untracked generated files in an outdated location, which fixes lint errors related to `src/generated` that may occur when switching branches.
2024-02-02 10:22:18 +01:00

123 lines
4.3 KiB
TypeScript

import type { AstId, Owned } from '.'
import { Ast, newExternalId } from '.'
import { assert } from '../util/assert'
import type { ExternalId } from '../yjsModel'
import { isUuid } from '../yjsModel'
import { is_ident_or_operator } from './ffi'
import * as RawAst from './generated/ast'
export function isToken(t: unknown): t is Token {
return t instanceof Token
}
declare const brandTokenId: unique symbol
export type TokenId = ExternalId & { [brandTokenId]: never }
function newTokenId(): TokenId {
return newExternalId() as TokenId
}
/** @internal */
export interface SyncTokenId {
readonly id: TokenId
code_: string
tokenType_: RawAst.Token.Type | undefined
}
export class Token implements SyncTokenId {
readonly id: TokenId
code_: string
tokenType_: RawAst.Token.Type | undefined
private constructor(code: string, type: RawAst.Token.Type | undefined, id: TokenId) {
this.id = id
this.code_ = code
this.tokenType_ = type
}
get externalId(): TokenId {
return this.id
}
static new(code: string, type?: RawAst.Token.Type) {
return new this(code, type, newTokenId())
}
static withId(code: string, type: RawAst.Token.Type | undefined, id: TokenId) {
assert(isUuid(id))
return new this(code, type, id)
}
code(): string {
return this.code_
}
typeName(): string {
if (this.tokenType_) return RawAst.Token.typeNames[this.tokenType_]!
else return 'Raw'
}
}
// We haven't had much need to distinguish token types, but it's useful to know that an identifier token's code is a
// valid string for an identifier.
export interface IdentifierOrOperatorIdentifierToken extends Token {
code(): IdentifierOrOperatorIdentifier
}
export interface IdentifierToken extends Token {
code(): Identifier
}
declare const qualifiedNameBrand: unique symbol
declare const identifierBrand: unique symbol
declare const operatorBrand: unique symbol
/** A string representing a valid qualified name of our language.
*
* In our language, the segments are separated by `.`. All the segments except the last must be lexical identifiers. The
* last may be an identifier or a lexical operator. A single identifier is also a valid qualified name.
*/
export type QualifiedName = string & { [qualifiedNameBrand]: never }
/** A string representing a lexical identifier. */
export type Identifier = string & { [identifierBrand]: never; [qualifiedNameBrand]: never }
/** A string representing a lexical operator. */
export type Operator = string & { [operatorBrand]: never; [qualifiedNameBrand]: never }
/** A string that can be parsed as an identifier in some contexts.
*
* If it is lexically an identifier (see `StrictIdentifier`), it can be used as identifier anywhere.
*
* If it is lexically an operator (see `Operator`), it takes the syntactic role of an identifier if it is the RHS of
* a `PropertyAccess`, or it is the name of a `Function` being defined within a type. In all other cases, it is not
* valid to use a lexical operator as an identifier (rather, it will usually parse as an `OprApp` or `UnaryOprApp`).
*/
export type IdentifierOrOperatorIdentifier = Identifier | Operator
/** Returns true if `code` can be used as an identifier in some contexts.
*
* If it is lexically an identifier (see `isIdentifier`), it can be used as identifier anywhere.
*
* If it is lexically an operator (see `isOperator`), it takes the syntactic role of an identifier if it is the RHS of
* a `PropertyAccess`, or it is the name of a `Function` being defined within a type. In all other cases, it is not
* valid to use a lexical operator as an identifier (rather, it will usually parse as an `OprApp` or `UnaryOprApp`).
*/
export function isIdentifierOrOperatorIdentifier(
code: string,
): code is IdentifierOrOperatorIdentifier {
return is_ident_or_operator(code) !== 0
}
/** Returns true if `code` is lexically an identifier. */
export function isIdentifier(code: string): code is Identifier {
return is_ident_or_operator(code) === 1
}
/** Returns true if `code` is lexically an operator. */
export function isOperator(code: string): code is Operator {
return is_ident_or_operator(code) === 2
}
/** @internal */
export function isTokenId(t: SyncTokenId | AstId | Ast | Owned<Ast> | Owned): t is SyncTokenId {
return typeof t === 'object' && !(t instanceof Ast)
}