Update the protocol to support streaming files (#1757)

2024-12-23 07:01:55 +03:00 · 2021-05-26 15:08:41 +01:00 · 2021-05-26 15:08:41 +01:00 · 3890abe6fa
commit 3890abe6fa
parent 80eff9c017
7 changed files with 450 additions and 8 deletions
--- a/distribution/std-lib/Standard/src/Visualization.enso
+++ b/distribution/std-lib/Standard/src/Visualization.enso
@ -0,0 +1,6 @@
+from Standard.Base import all
+
+import Standard.Visualization.File_Upload
+
+from Standard.Visualization.File_Upload export file_uploading
+
--- a/distribution/std-lib/Standard/src/Visualization/File_Upload.enso
+++ b/distribution/std-lib/Standard/src/Visualization/File_Upload.enso
@ -0,0 +1,26 @@
+from Standard.Base import all
+
+import Standard.Base.System.File as Base_File
+
+## UNSTABLE
+
+   A function that throws an error to indicate that a file is being uploaded to
+   `path`.
+
+   Arguments:
+   - `path`: The path to which the file is being uploaded.
+file_uploading : (Base_File.File | Text) -> Base_File.File ! File_Being_Uploaded
+file_uploading path =
+    err = File_Being_Uploaded <| case path of
+        Text -> path
+        Base_File.File _ -> path.path
+        _ -> ""
+    Error.throw err
+
+## UNSTABLE
+
+   Represents that a file is being uploaded to the given `file_path`.
+
+   Arguments:
+   - file_path: The path at which the file is being uploaded.
+type File_Being_Uploaded file_path
--- a/docs/language-server/README.md
+++ b/docs/language-server/README.md
@ -34,6 +34,8 @@ is broken up as follows:

 - [**The Enso Protocol Architecture:**](./protocol-architecture.md) The
  architecture of the Enso protocol.
+- [**Streaming File Transfer:**](./streaming-file-transfer.md) Documentation on
+  how the streaming file transfer mechanism works.

 The protocol messages are broken up into documents as follows:

--- a/docs/language-server/protocol-language-server.md
+++ b/docs/language-server/protocol-language-server.md
@ -23,9 +23,9 @@ transport formats, please look [here](./protocol-architecture).
  - [`ContextId`](#contextid)
  - [`StackItem`](#stackitem)
  - [`MethodPointer`](#methodpointer)
+  - [`ProfilingInfo`](#profilinginfo)
  - [`ExpressionUpdate`](#expressionupdate)
  - [`ExpressionUpdatePayload`](#expressionupdatepayload)
-  - [`ProfilingInfo`](#profilinginfo)
  - [`VisualisationConfiguration`](#visualisationconfiguration)
  - [`SuggestionEntryArgument`](#suggestionentryargument)
  - [`SuggestionEntry`](#suggestionentry)
@ -53,6 +53,8 @@ transport formats, please look [here](./protocol-architecture).
  - [`FileContents`](#filecontents)
  - [`FileSystemObject`](#filesystemobject)
  - [`WorkspaceEdit`](#workspaceedit)
+  - [`EnsoDigest`](#ensodigest)
+  - [`FileSegment`](#filesegment)
 - [Connection Management](#connection-management)
  - [`session/initProtocolConnection`](#sessioninitprotocolconnection)
  - [`session/initBinaryConnection`](#sessioninitbinaryconnection)
@ -67,13 +69,13 @@ transport formats, please look [here](./protocol-architecture).
  - [`executionContext/canModify`](#executioncontextcanmodify)
  - [`executionContext/receivesUpdates`](#executioncontextreceivesupdates)
  - [`search/receivesSuggestionsDatabaseUpdates`](#searchreceivessuggestionsdatabaseupdates)
-  - [Enables](#enables)
-  - [Disables](#disables)
 - [File Management Operations](#file-management-operations)
  - [`file/write`](#filewrite)
  - [`file/read`](#fileread)
  - [`file/writeBinary`](#filewritebinary)
  - [`file/readBinary`](#filereadbinary)
+  - [`file/writeBytes`](#filewritebytes)
+  - [`file/readBytes`](#filereadbytes)
  - [`file/create`](#filecreate)
  - [`file/delete`](#filedelete)
  - [`file/copy`](#filecopy)
@ -82,6 +84,8 @@ transport formats, please look [here](./protocol-architecture).
  - [`file/tree`](#filetree)
  - [`file/list`](#filelist)
  - [`file/info`](#fileinfo)
+  - [`file/checksum`](#filechecksum)
+  - [`file/checksumBytes`](#filechecksumbytes)
  - [`file/event`](#fileevent)
  - [`file/addRoot`](#fileaddroot)
  - [`file/removeRoot`](#fileremoveroot)
@ -139,6 +143,7 @@ transport formats, please look [here](./protocol-architecture).
  - [`io/feedStandardInput`](#iofeedstandardinput)
  - [`io/waitingForStandardInput`](#iowaitingforstandardinput)
 - [Errors](#errors)
+  - [`Error`](#error)
  - [`AccessDeniedError`](#accessdeniederror)
  - [`FileSystemError`](#filesystemerror)
  - [`ContentRootNotFoundError`](#contentrootnotfounderror)
@ -146,6 +151,9 @@ transport formats, please look [here](./protocol-architecture).
  - [`FileExists`](#fileexists-1)
  - [`OperationTimeoutError`](#operationtimeouterror)
  - [`NotDirectory`](#notdirectory)
+  - [`NotFile`](#notfile)
+  - [`CannotOverwrite`](#cannotoverwrite)
+  - [`ReadOutOfBounds`](#readoutofbounds)
  - [`StackItemNotFoundError`](#stackitemnotfounderror)
  - [`ContextNotFoundError`](#contextnotfounderror)
  - [`EmptyStackError`](#emptystackerror)
@ -717,8 +725,7 @@ may be expanded in future.
 * @param creationTime creation time
 * @param lastAccessTime last access time
 * @param lastModifiedTime last modified time
- * @param kind type of [[FileSystemObject]], can be:
- * `Directory`, `File`, `Other`
+ * @param kind type of [[FileSystemObject]], can be: `Directory`, `File`, `Other`
 * @param byteSize size in bytes
 */
 interface FileAttributes {
@ -906,7 +913,8 @@ interface Diagnostic {

 ### `SHA3-224`

-The `SHA3-224` message digest encoded as a base16 string.
+The `SHA3-224` message digest encoded as a base16 string. For the equivalent
+structure on the binary connection please see [`EnsoDigest`](#ensodigest)

 #### Format

@ -1027,6 +1035,46 @@ undo/redo.
 > - Work out the design of this message.
 > - Specify this message.

+### `EnsoDigest`
+
+A counterpart to [SHA3-224](#sha3-224) for the binary connection, this is a
+standard message digest encoded using FlatBuffers.
+
+```idl
+namespace org.enso.languageserver.protocol.binary;
+
+table EnsoDigest {
+  bytes : [ubyte];
+}
+```
+
+Notes:
+
+- It is an error for the length of the vector `bytes` to not be equal to 28 (224
+  / 8). This is the length of the chosen digest in bytes.
+
+### `FileSegment`
+
+A representation of a segment of a file for use in the binary protocol.
+
+```idl
+namespace org.enso.languageserver.protocol.binary;
+
+table FileSegment {
+  // The file to access.
+  path : Path (required);
+
+  // The byte offset in the file to read from.
+  byteOffset : ulong (required);
+
+  // The number of bytes to read.
+  length : ulong (required);
+}
+```
+
+The `byteOffset` property is zero-indexed, so the last byte in the file is at
+index `file.length - 1`.
+
 ## Connection Management

 In order to properly set-up and tear-down the language server connection, we
@ -1318,11 +1366,11 @@ a given execution context.
 - **method:** `search/receivesSuggestionsDatabaseUpdates`
 - **registerOptions:** `{}`

-### Enables
+#### Enables

 - [`search/suggestionsDatabaseUpdate`](#suggestionsdatabaseupdate)

-### Disables
+#### Disables

 None

@ -1507,6 +1555,118 @@ table FileContentsReply {
  access to a resource.
 - [`FileNotFound`](#filenotfound) informs that file cannot be found.

+### `file/writeBytes`
+
+This requests that the file manager component writes a set of bytes to the
+specified file at the specified offset.
+
+- **Type:** Request
+- **Direction:** Client -> Server
+- **Connection:** Binary
+- **Visibility:** Public
+
+This method will create a file if no file is present at `path`.
+
+- The `overwriteExisting` boolean should be set if `byteOffset` is less than the
+  length of the file.
+- The `byteOffset` property is zero-indexed. To append to the file you begin
+  writing at index `file.length`.
+- If `byteOffset > file.length`, the bytes in the range
+  `[file.length, byteOffset)` will be filled with null bytes.
+
+#### Parameters
+
+```idl
+namespace org.enso.languageserver.protocol.binary;
+
+table WriteBytesRequest {
+  // The file to write to.
+  path : Path (required);
+
+  // The byte offset in the file to write from.
+  byteOffset : ulong (required);
+
+  // Whether existing content should be overwritten.
+  overwriteExisting : bool (required);
+
+  // The file contents.
+  bytes : [ubyte] (required);
+}
+```
+
+#### Result
+
+```idl
+namespace org.enso.languageserver.protocol.binary;
+
+table WriteBytesResponse {
+  // The checksum of the written bytes.
+  checksum : EnsoDigest (required);
+}
+```
+
+Notes:
+
+- The `checksum` is only of the `bytes` in the request as they were written to
+  disk. This does _not_ include checksumming the entire file. For that, please
+  see [`file/checksumBytes`](#file-checksumbytes).
+
+#### Errors
+
+- [`CannotOverwrite`](#cannotoverwrite) to signal that an overwrite would be
+  necessary to perform the operation but that `overwriteExisting` is not set.
+- [`NotFile`](#notfile) if the provided `segment.path` is not a file.
+
+### `file/readBytes`
+
+Asks the language server to read the specified number of bytes at the specified
+offset in the file.
+
+- **Type:** Request
+- **Direction:** Client -> Server
+- **Connection:** Binary
+- **Visibility:** Public
+
+It will attempt to read _as many as_ `segment.length` bytes, but does not
+guarantee that the response will contain `segment.length` bytes (e.g. if
+`segment.length` would require reading off the end of the file).
+
+#### Parameters
+
+```idl
+namespace org.enso.languageserver.protocol.binary;
+
+table ReadBytesRequest {
+  // The segment in a file to read bytes from.
+  segment : FileSegment (required);
+}
+```
+
+#### Result
+
+```idl
+namespace org.enso.languageserver.protocol.binary;
+
+table ReadBytesResponse {
+  // The checksum of the bytes in this response.
+  checksum : EnsoDigest (required);
+
+  // The requested file contents.
+  bytes : [ubyte] (required);
+}
+```
+
+Notes:
+
+- The `checksum` is of the `bytes` as they have been read from disk.
+
+#### Errors
+
+- [`FileNotFound`](#filenotfound) if the file at `segment.path` does not exist.
+- [`ReadOutOfBounds`](#readoutofbounds) if `segment.byteOffset` is not present
+  in the file at `segment.path`.
+- [`NotFile`](#notfile) if the provided `segment.path` is not a file.
+
 ### `file/create`

 This request asks the file manager to create the specified file system object.
@ -1778,6 +1938,81 @@ This request should work for all kinds of filesystem object.
  requested content root cannot be found.
 - [`FileNotFound`](#filenotfound) informs that requested path does not exist.

+### `file/checksum`
+
+Requests that the language server provide the checksum of the provided file.
+Only defined when the provided `path` is a file.
+
+- **Type:** Request
+- **Direction:** Client -> Server
+- **Connection:** Protocol
+- **Visibility:** Public
+
+It calculates the checksum of the entire file.
+
+#### Parameters
+
+```typescript
+interface ChecksumRequest {
+  // The path to the file to get the checksum for.
+  path: Path;
+}
+```
+
+#### Result
+
+```typescript
+interface ChecksumResponse {
+  // The checksum of the file at `path`.
+  checksum : SHA3-224;
+}
+```
+
+#### Errors
+
+- [`FileNotFound`](#filenotfound) if the file at `path` does not exist.
+- [`NotFile`](#notfile) if the provided `path` does not point to a file.
+
+### `file/checksumBytes`
+
+Requests that the language server provides the checksum of the provided byte
+range.
+
+- **Type:** Request
+- **Direction:** Client -> Server
+- **Connection:** Binary
+- **Visibility:** Public
+
+#### Parameters
+
+```idl
+namespace org.enso.languageserver.protocol.binary;
+
+table ChecksumBytesRequest {
+  // The segment in a file to checksum.
+  segment : FileSegment (required);
+}
+```
+
+#### Result
+
+```idl
+namespace org.enso.languageserver.protocol.binary;
+
+table ChecksumBytesRequest {
+  // The segment in a file to checksum.
+  checksum : EnsoDigest;
+}
+```
+
+#### Errors
+
+- [`FileNotFound`](#filenotfound) if the file at `segment.path` does not exist.
+- [`ReadOutOfBounds`](#readoutofbounds) if `segment.byteOffset` is not present
+  in the file at `segment.path`, or if `segment.length` does not fit within the
+  file.
+- [`NotFile`](#notfile) if the provided `segment.path` is not a file.
+
 ### `file/event`

 This is a notification that is sent every time something under a watched content
@ -3548,6 +3783,39 @@ not a complete specification and will be updated as new errors are added.
 Besides the required `code` and `message` fields, the errors may have a `data`
 field which can store additional error-specific payload.

+### `Error`
+
+An error container for the binary connection that contains a code, message and
+payload.
+
+```idl
+namespace org.enso.languageserver.protocol.binary;
+
+table Error {
+  // A unique error code identifying error type.
+  code: int (required);
+
+  // An error message.
+  message: string (required);
+
+  // Additional payloads for the error.
+  data : ErrorPayload (required);
+}
+
+union ErrorPayload {
+  EMPTY: EmptyPayload,
+  ...
+}
+
+struct EmptyPayload {}
+```
+
+Note:
+
+- The union `ErrorPayload` will be extended with additional payloads as
+  necessary.
+- All textual-protocol errors can be represented using this structure.
+
 ### `AccessDeniedError`

 It signals that a user doesn't have access to a resource.
@ -3625,6 +3893,52 @@ It signals that provided path is not a directory.
 }
 ```

+### `NotFile`
+
+It signals that the provided path is not a file.
+
+```typescript
+"error" : {
+  "code" : 1007,
+  "message" : "Path is not a file"
+}
+```
+
+### `CannotOverwrite`
+
+Signals that a streaming file write cannot overwrite a portion of the requested
+file.
+
+```typescript
+"error" : {
+  "code" : 1008,
+  "message" : "Cannot overwrite the file without `overwriteExisting` set"
+}
+```
+
+### `ReadOutOfBounds`
+
+Signals that the requested file read was out of bounds for the file's size.
+
+```typescript
+"error" : {
+  "code" : 1009
+  "message" : "Read is out of bounds for the file"
+  "data" : {
+    fileLength : 0
+  }
+}
+```
+
+```idl
+namespace org.enso.languageserver.protocol.binary;
+
+table ReadOutOfBoundsError {
+  // The actual length of the file.
+  fileLength : ulong (required);
+}
+```
+
 ### `StackItemNotFoundError`

 It signals that provided stack item was not found.
--- a/docs/language-server/streaming-file-transfer.md
+++ b/docs/language-server/streaming-file-transfer.md
@ -0,0 +1,72 @@
+---
+layout: developer-doc
+title: Streaming File Transfer
+category: language-server
+tags: [language-server, protocol, specification]
+order: 5
+---
+
+# Streaming File Transfer
+
+Particularly important in the separate-server design that we provide with the
+Language Server is the ability to transfer files to and from the remote machine.
+To that end, it is important that we provide the ability for the IDE to both
+upload and download very large files.
+
+A few key requirements:
+
+- We want to support resumption of transfers.
+- We want to have transfers be as low-overhead as possible.
+- Multiple transfers may be occurring at once.
+- We want to keep implementation simple to avoid errors (ideally no state).
+
+<!-- MarkdownTOC levels="2,3" autolink="true" indent="    " -->
+
+- [Control](#control)
+- [UX](#ux)
+
+<!-- /MarkdownTOC -->
+
+## Control
+
+In order to make this portion of the protocol simple to manage, it is defined in
+a stateless fashion. The Language Server provides two messages `file/writeBytes`
+and `file/readByteRange` with corresponding responses. We make the following
+assumptions:
+
+- Each request must be completed with a response before sending another request
+  for the same file.
+- Basic file information can be provided by the existing `file/info` API, which
+  allows the IDE to create progress spinners and other such niceties.
+- All requests and responses are contained within the `InboundMessage` and
+  `OutboundMessage` containers respectively.
+
+The assumption is that, rather than encoding a stateful protocol, we instead
+rely on the IDE to control the upload and download of files by sending
+successive requests. To upload files, we intend for the IDE to use
+`file/writeBytes`, and to download files we intend for `file/readBytes` to be
+used.
+
+Resumption of transfers is also handled by the IDE, which may keep track of what
+portions of a file have been written or read.
+
+## UX
+
+The IDE wants to be able to provide two major UX benefits to users as part of
+this work:
+
+1. Loading spinners that show the progress of the file.
+2. A display of parts of the graph that are waiting on the file.
+
+The first requirement here is trivially handled due to the IDE-driven nature of
+the file upload and download process. As they know the size of the file being
+transferred and get acknowledgements of each chunk of the file that is sent,
+they know both the speed and the amount of the file that has been uploaded.
+
+The second requirement is being handled by the addition of the
+`Visualization.file_uploading` method to the `Visualization` portion of the
+standard library. This is a method that returns a dataflow error
+`File_Being_Uploaded path` that will flow through the graph to annotate all
+portions waiting on the file upload. All the IDE has to do is insert this
+expression implicitly into the source file while the upload is progressing, and
+it can trace the impacted nodes and display the necessary UI details.
--- a/test/Visualization_Tests/src/Main.enso
+++ b/test/Visualization_Tests/src/Main.enso
@ -8,6 +8,7 @@ import Visualization_Tests.Histogram_Spec
 import Visualization_Tests.Scatter_Plot_Spec
 import Visualization_Tests.Sql_Spec
 import Visualization_Tests.Table_Spec
+import Visualization_Tests.Visualization_Spec

 main = Test.Suite.run_main <|
    Geo_Map_Spec.spec
@ -16,3 +17,4 @@ main = Test.Suite.run_main <|
    Scatter_Plot_Spec.spec
    Sql_Spec.spec
    Table_Spec.spec
+    Visualization_Spec.spec
--- a/test/Visualization_Tests/src/Visualization_Spec.enso
+++ b/test/Visualization_Tests/src/Visualization_Spec.enso
@ -0,0 +1,20 @@
+from Standard.Base import all
+
+import Standard.Examples
+import Standard.Test
+import Standard.Visualization
+
+from Standard.Visualization.File_Upload import File_Being_Uploaded
+
+spec = Test.group "File uploads" <|
+    Test.specify "should be able to be signalled as uploading" <|
+        Visualization.file_uploading "file" . should_fail_with File_Being_Uploaded
+
+    Test.specify "should work whether a textual or file path is provided" <|
+        result_file = Visualization.file_uploading Examples.csv . catch
+        result_file.file_path . should_equal Examples.csv_path
+
+        result_text = Visualization.file_uploading Examples.csv_path . catch
+        result_text.file_path . should_equal Examples.csv_path
+
+