mirror of
https://github.com/enso-org/enso.git
synced 2024-12-23 10:42:05 +03:00
Update the protocol to support streaming files (#1757)
This commit is contained in:
parent
80eff9c017
commit
3890abe6fa
6
distribution/std-lib/Standard/src/Visualization.enso
Normal file
6
distribution/std-lib/Standard/src/Visualization.enso
Normal file
@ -0,0 +1,6 @@
|
||||
from Standard.Base import all
|
||||
|
||||
import Standard.Visualization.File_Upload
|
||||
|
||||
from Standard.Visualization.File_Upload export file_uploading
|
||||
|
@ -0,0 +1,26 @@
|
||||
from Standard.Base import all
|
||||
|
||||
import Standard.Base.System.File as Base_File
|
||||
|
||||
## UNSTABLE
|
||||
|
||||
A function that throws an error to indicate that a file is being uploaded to
|
||||
`path`.
|
||||
|
||||
Arguments:
|
||||
- `path`: The path to which the file is being uploaded.
|
||||
file_uploading : (Base_File.File | Text) -> Base_File.File ! File_Being_Uploaded
|
||||
file_uploading path =
|
||||
err = File_Being_Uploaded <| case path of
|
||||
Text -> path
|
||||
Base_File.File _ -> path.path
|
||||
_ -> ""
|
||||
Error.throw err
|
||||
|
||||
## UNSTABLE
|
||||
|
||||
Represents that a file is being uploaded to the given `file_path`.
|
||||
|
||||
Arguments:
|
||||
- file_path: The path at which the file is being uploaded.
|
||||
type File_Being_Uploaded file_path
|
@ -34,6 +34,8 @@ is broken up as follows:
|
||||
|
||||
- [**The Enso Protocol Architecture:**](./protocol-architecture.md) The
|
||||
architecture of the Enso protocol.
|
||||
- [**Streaming File Transfer:**](./streaming-file-transfer.md) Documentation on
|
||||
how the streaming file transfer mechanism works.
|
||||
|
||||
The protocol messages are broken up into documents as follows:
|
||||
|
||||
|
@ -23,9 +23,9 @@ transport formats, please look [here](./protocol-architecture).
|
||||
- [`ContextId`](#contextid)
|
||||
- [`StackItem`](#stackitem)
|
||||
- [`MethodPointer`](#methodpointer)
|
||||
- [`ProfilingInfo`](#profilinginfo)
|
||||
- [`ExpressionUpdate`](#expressionupdate)
|
||||
- [`ExpressionUpdatePayload`](#expressionupdatepayload)
|
||||
- [`ProfilingInfo`](#profilinginfo)
|
||||
- [`VisualisationConfiguration`](#visualisationconfiguration)
|
||||
- [`SuggestionEntryArgument`](#suggestionentryargument)
|
||||
- [`SuggestionEntry`](#suggestionentry)
|
||||
@ -53,6 +53,8 @@ transport formats, please look [here](./protocol-architecture).
|
||||
- [`FileContents`](#filecontents)
|
||||
- [`FileSystemObject`](#filesystemobject)
|
||||
- [`WorkspaceEdit`](#workspaceedit)
|
||||
- [`EnsoDigest`](#ensodigest)
|
||||
- [`FileSegment`](#filesegment)
|
||||
- [Connection Management](#connection-management)
|
||||
- [`session/initProtocolConnection`](#sessioninitprotocolconnection)
|
||||
- [`session/initBinaryConnection`](#sessioninitbinaryconnection)
|
||||
@ -67,13 +69,13 @@ transport formats, please look [here](./protocol-architecture).
|
||||
- [`executionContext/canModify`](#executioncontextcanmodify)
|
||||
- [`executionContext/receivesUpdates`](#executioncontextreceivesupdates)
|
||||
- [`search/receivesSuggestionsDatabaseUpdates`](#searchreceivessuggestionsdatabaseupdates)
|
||||
- [Enables](#enables)
|
||||
- [Disables](#disables)
|
||||
- [File Management Operations](#file-management-operations)
|
||||
- [`file/write`](#filewrite)
|
||||
- [`file/read`](#fileread)
|
||||
- [`file/writeBinary`](#filewritebinary)
|
||||
- [`file/readBinary`](#filereadbinary)
|
||||
- [`file/writeBytes`](#filewritebytes)
|
||||
- [`file/readBytes`](#filereadbytes)
|
||||
- [`file/create`](#filecreate)
|
||||
- [`file/delete`](#filedelete)
|
||||
- [`file/copy`](#filecopy)
|
||||
@ -82,6 +84,8 @@ transport formats, please look [here](./protocol-architecture).
|
||||
- [`file/tree`](#filetree)
|
||||
- [`file/list`](#filelist)
|
||||
- [`file/info`](#fileinfo)
|
||||
- [`file/checksum`](#filechecksum)
|
||||
- [`file/checksumBytes`](#filechecksumbytes)
|
||||
- [`file/event`](#fileevent)
|
||||
- [`file/addRoot`](#fileaddroot)
|
||||
- [`file/removeRoot`](#fileremoveroot)
|
||||
@ -139,6 +143,7 @@ transport formats, please look [here](./protocol-architecture).
|
||||
- [`io/feedStandardInput`](#iofeedstandardinput)
|
||||
- [`io/waitingForStandardInput`](#iowaitingforstandardinput)
|
||||
- [Errors](#errors)
|
||||
- [`Error`](#error)
|
||||
- [`AccessDeniedError`](#accessdeniederror)
|
||||
- [`FileSystemError`](#filesystemerror)
|
||||
- [`ContentRootNotFoundError`](#contentrootnotfounderror)
|
||||
@ -146,6 +151,9 @@ transport formats, please look [here](./protocol-architecture).
|
||||
- [`FileExists`](#fileexists-1)
|
||||
- [`OperationTimeoutError`](#operationtimeouterror)
|
||||
- [`NotDirectory`](#notdirectory)
|
||||
- [`NotFile`](#notfile)
|
||||
- [`CannotOverwrite`](#cannotoverwrite)
|
||||
- [`ReadOutOfBounds`](#readoutofbounds)
|
||||
- [`StackItemNotFoundError`](#stackitemnotfounderror)
|
||||
- [`ContextNotFoundError`](#contextnotfounderror)
|
||||
- [`EmptyStackError`](#emptystackerror)
|
||||
@ -717,8 +725,7 @@ may be expanded in future.
|
||||
* @param creationTime creation time
|
||||
* @param lastAccessTime last access time
|
||||
* @param lastModifiedTime last modified time
|
||||
* @param kind type of [[FileSystemObject]], can be:
|
||||
* `Directory`, `File`, `Other`
|
||||
* @param kind type of [[FileSystemObject]], can be: `Directory`, `File`, `Other`
|
||||
* @param byteSize size in bytes
|
||||
*/
|
||||
interface FileAttributes {
|
||||
@ -906,7 +913,8 @@ interface Diagnostic {
|
||||
|
||||
### `SHA3-224`
|
||||
|
||||
The `SHA3-224` message digest encoded as a base16 string.
|
||||
The `SHA3-224` message digest encoded as a base16 string. For the equivalent
|
||||
structure on the binary connection please see [`EnsoDigest`](#ensodigest)
|
||||
|
||||
#### Format
|
||||
|
||||
@ -1027,6 +1035,46 @@ undo/redo.
|
||||
> - Work out the design of this message.
|
||||
> - Specify this message.
|
||||
|
||||
### `EnsoDigest`
|
||||
|
||||
A counterpart to [SHA3-224](#sha3-224) for the binary connection, this is a
|
||||
standard message digest encoded using FlatBuffers.
|
||||
|
||||
```idl
|
||||
namespace org.enso.languageserver.protocol.binary;
|
||||
|
||||
table EnsoDigest {
|
||||
bytes : [ubyte];
|
||||
}
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- It is an error for the length of the vector `bytes` to not be equal to 28 (224
|
||||
/ 8). This is the length of the chosen digest in bytes.
|
||||
|
||||
### `FileSegment`
|
||||
|
||||
A representation of a segment of a file for use in the binary protocol.
|
||||
|
||||
```idl
|
||||
namespace org.enso.languageserver.protocol.binary;
|
||||
|
||||
table FileSegment {
|
||||
// The file to access.
|
||||
path : Path (required);
|
||||
|
||||
// The byte offset in the file to read from.
|
||||
byteOffset : ulong (required);
|
||||
|
||||
// The number of bytes to read.
|
||||
length : ulong (required);
|
||||
}
|
||||
```
|
||||
|
||||
The `byteOffset` property is zero-indexed, so the last byte in the file is at
|
||||
index `file.length - 1`.
|
||||
|
||||
## Connection Management
|
||||
|
||||
In order to properly set-up and tear-down the language server connection, we
|
||||
@ -1318,11 +1366,11 @@ a given execution context.
|
||||
- **method:** `search/receivesSuggestionsDatabaseUpdates`
|
||||
- **registerOptions:** `{}`
|
||||
|
||||
### Enables
|
||||
#### Enables
|
||||
|
||||
- [`search/suggestionsDatabaseUpdate`](#suggestionsdatabaseupdate)
|
||||
|
||||
### Disables
|
||||
#### Disables
|
||||
|
||||
None
|
||||
|
||||
@ -1507,6 +1555,118 @@ table FileContentsReply {
|
||||
access to a resource.
|
||||
- [`FileNotFound`](#filenotfound) informs that file cannot be found.
|
||||
|
||||
### `file/writeBytes`
|
||||
|
||||
This requests that the file manager component writes a set of bytes to the
|
||||
specified file at the specified offset.
|
||||
|
||||
- **Type:** Request
|
||||
- **Direction:** Client -> Server
|
||||
- **Connection:** Binary
|
||||
- **Visibility:** Public
|
||||
|
||||
This method will create a file if no file is present at `path`.
|
||||
|
||||
- The `overwriteExisting` boolean should be set if `byteOffset` is less than the
|
||||
length of the file.
|
||||
- The `byteOffset` property is zero-indexed. To append to the file you begin
|
||||
writing at index `file.length`.
|
||||
- If `byteOffset > file.length`, the bytes in the range
|
||||
`[file.length, byteOffset)` will be filled with null bytes.
|
||||
|
||||
#### Parameters
|
||||
|
||||
```idl
|
||||
namespace org.enso.languageserver.protocol.binary;
|
||||
|
||||
table WriteBytesRequest {
|
||||
// The file to write to.
|
||||
path : Path (required);
|
||||
|
||||
// The byte offset in the file to write from.
|
||||
byteOffset : ulong (required);
|
||||
|
||||
// Whether existing content should be overwritten.
|
||||
overwriteExisting : bool (required);
|
||||
|
||||
// The file contents.
|
||||
bytes : [ubyte] (required);
|
||||
}
|
||||
```
|
||||
|
||||
#### Result
|
||||
|
||||
```idl
|
||||
namespace org.enso.languageserver.protocol.binary;
|
||||
|
||||
table WriteBytesResponse {
|
||||
// The checksum of the written bytes.
|
||||
checksum : EnsoDigest (required);
|
||||
}
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- The `checksum` is only of the `bytes` in the request as they were written to
|
||||
disk. This does _not_ include checksumming the entire file. For that, please
|
||||
see [`file/checksumBytes`](#file-checksumbytes).
|
||||
|
||||
#### Errors
|
||||
|
||||
- [`CannotOverwrite`](#cannotoverwrite) to signal that an overwrite would be
|
||||
necessary to perform the operation but that `overwriteExisting` is not set.
|
||||
- [`NotFile`](#notfile) if the provided `segment.path` is not a file.
|
||||
|
||||
### `file/readBytes`
|
||||
|
||||
Asks the language server to read the specified number of bytes at the specified
|
||||
offset in the file.
|
||||
|
||||
- **Type:** Request
|
||||
- **Direction:** Client -> Server
|
||||
- **Connection:** Binary
|
||||
- **Visibility:** Public
|
||||
|
||||
It will attempt to read _as many as_ `segment.length` bytes, but does not
|
||||
guarantee that the response will contain `segment.length` bytes (e.g. if
|
||||
`segment.length` would require reading off the end of the file).
|
||||
|
||||
#### Parameters
|
||||
|
||||
```idl
|
||||
namespace org.enso.languageserver.protocol.binary;
|
||||
|
||||
table ReadBytesRequest {
|
||||
// The segment in a file to read bytes from.
|
||||
segment : FileSegment (required);
|
||||
}
|
||||
```
|
||||
|
||||
#### Result
|
||||
|
||||
```idl
|
||||
namespace org.enso.languageserver.protocol.binary;
|
||||
|
||||
table ReadBytesResponse {
|
||||
// The checksum of the bytes in this response.
|
||||
checksum : EnsoDigest (required);
|
||||
|
||||
// The requested file contents.
|
||||
bytes : [ubyte] (required);
|
||||
}
|
||||
```
|
||||
|
||||
Notes:
|
||||
|
||||
- The `checksum` is of the `bytes` as they have been read from disk.
|
||||
|
||||
#### Errors
|
||||
|
||||
- [`FileNotFound`](#filenotfound) if the file at `segment.path` does not exist.
|
||||
- [`ReadOutOfBounds`](#readoutofbounds) if `segment.byteOffset` is not present
|
||||
in the file at `segment.path`.
|
||||
- [`NotFile`](#notfile) if the provided `segment.path` is not a file.
|
||||
|
||||
### `file/create`
|
||||
|
||||
This request asks the file manager to create the specified file system object.
|
||||
@ -1778,6 +1938,81 @@ This request should work for all kinds of filesystem object.
|
||||
requested content root cannot be found.
|
||||
- [`FileNotFound`](#filenotfound) informs that requested path does not exist.
|
||||
|
||||
### `file/checksum`
|
||||
|
||||
Requests that the language server provide the checksum of the provided file.
|
||||
Only defined when the provided `path` is a file.
|
||||
|
||||
- **Type:** Request
|
||||
- **Direction:** Client -> Server
|
||||
- **Connection:** Protocol
|
||||
- **Visibility:** Public
|
||||
|
||||
It calculates the checksum of the entire file.
|
||||
|
||||
#### Parameters
|
||||
|
||||
```typescript
|
||||
interface ChecksumRequest {
|
||||
// The path to the file to get the checksum for.
|
||||
path: Path;
|
||||
}
|
||||
```
|
||||
|
||||
#### Result
|
||||
|
||||
```typescript
|
||||
interface ChecksumResponse {
|
||||
// The checksum of the file at `path`.
|
||||
checksum : SHA3-224;
|
||||
}
|
||||
```
|
||||
|
||||
#### Errors
|
||||
|
||||
- [`FileNotFound`](#filenotfound) if the file at `path` does not exist.
|
||||
- [`NotFile`](#notfile) if the provided `path` does not point to a file.
|
||||
|
||||
### `file/checksumBytes`
|
||||
|
||||
Requests that the language server provides the checksum of the provided byte
|
||||
range.
|
||||
|
||||
- **Type:** Request
|
||||
- **Direction:** Client -> Server
|
||||
- **Connection:** Binary
|
||||
- **Visibility:** Public
|
||||
|
||||
#### Parameters
|
||||
|
||||
```idl
|
||||
namespace org.enso.languageserver.protocol.binary;
|
||||
|
||||
table ChecksumBytesRequest {
|
||||
// The segment in a file to checksum.
|
||||
segment : FileSegment (required);
|
||||
}
|
||||
```
|
||||
|
||||
#### Result
|
||||
|
||||
```idl
|
||||
namespace org.enso.languageserver.protocol.binary;
|
||||
|
||||
table ChecksumBytesRequest {
|
||||
// The segment in a file to checksum.
|
||||
checksum : EnsoDigest;
|
||||
}
|
||||
```
|
||||
|
||||
#### Errors
|
||||
|
||||
- [`FileNotFound`](#filenotfound) if the file at `segment.path` does not exist.
|
||||
- [`ReadOutOfBounds`](#readoutofbounds) if `segment.byteOffset` is not present
|
||||
in the file at `segment.path`, or if `segment.length` does not fit within the
|
||||
file.
|
||||
- [`NotFile`](#notfile) if the provided `segment.path` is not a file.
|
||||
|
||||
### `file/event`
|
||||
|
||||
This is a notification that is sent every time something under a watched content
|
||||
@ -3548,6 +3783,39 @@ not a complete specification and will be updated as new errors are added.
|
||||
Besides the required `code` and `message` fields, the errors may have a `data`
|
||||
field which can store additional error-specific payload.
|
||||
|
||||
### `Error`
|
||||
|
||||
An error container for the binary connection that contains a code, message and
|
||||
payload.
|
||||
|
||||
```idl
|
||||
namespace org.enso.languageserver.protocol.binary;
|
||||
|
||||
table Error {
|
||||
// A unique error code identifying error type.
|
||||
code: int (required);
|
||||
|
||||
// An error message.
|
||||
message: string (required);
|
||||
|
||||
// Additional payloads for the error.
|
||||
data : ErrorPayload (required);
|
||||
}
|
||||
|
||||
union ErrorPayload {
|
||||
EMPTY: EmptyPayload,
|
||||
...
|
||||
}
|
||||
|
||||
struct EmptyPayload {}
|
||||
```
|
||||
|
||||
Note:
|
||||
|
||||
- The union `ErrorPayload` will be extended with additional payloads as
|
||||
necessary.
|
||||
- All textual-protocol errors can be represented using this structure.
|
||||
|
||||
### `AccessDeniedError`
|
||||
|
||||
It signals that a user doesn't have access to a resource.
|
||||
@ -3625,6 +3893,52 @@ It signals that provided path is not a directory.
|
||||
}
|
||||
```
|
||||
|
||||
### `NotFile`
|
||||
|
||||
It signals that the provided path is not a file.
|
||||
|
||||
```typescript
|
||||
"error" : {
|
||||
"code" : 1007,
|
||||
"message" : "Path is not a file"
|
||||
}
|
||||
```
|
||||
|
||||
### `CannotOverwrite`
|
||||
|
||||
Signals that a streaming file write cannot overwrite a portion of the requested
|
||||
file.
|
||||
|
||||
```typescript
|
||||
"error" : {
|
||||
"code" : 1008,
|
||||
"message" : "Cannot overwrite the file without `overwriteExisting` set"
|
||||
}
|
||||
```
|
||||
|
||||
### `ReadOutOfBounds`
|
||||
|
||||
Signals that the requested file read was out of bounds for the file's size.
|
||||
|
||||
```typescript
|
||||
"error" : {
|
||||
"code" : 1009
|
||||
"message" : "Read is out of bounds for the file"
|
||||
"data" : {
|
||||
fileLength : 0
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
```idl
|
||||
namespace org.enso.languageserver.protocol.binary;
|
||||
|
||||
table ReadOutOfBoundsError {
|
||||
// The actual length of the file.
|
||||
fileLength : ulong (required);
|
||||
}
|
||||
```
|
||||
|
||||
### `StackItemNotFoundError`
|
||||
|
||||
It signals that provided stack item was not found.
|
||||
|
72
docs/language-server/streaming-file-transfer.md
Normal file
72
docs/language-server/streaming-file-transfer.md
Normal file
@ -0,0 +1,72 @@
|
||||
---
|
||||
layout: developer-doc
|
||||
title: Streaming File Transfer
|
||||
category: language-server
|
||||
tags: [language-server, protocol, specification]
|
||||
order: 5
|
||||
---
|
||||
|
||||
# Streaming File Transfer
|
||||
|
||||
Particularly important in the separate-server design that we provide with the
|
||||
Language Server is the ability to transfer files to and from the remote machine.
|
||||
To that end, it is important that we provide the ability for the IDE to both
|
||||
upload and download very large files.
|
||||
|
||||
A few key requirements:
|
||||
|
||||
- We want to support resumption of transfers.
|
||||
- We want to have transfers be as low-overhead as possible.
|
||||
- Multiple transfers may be occurring at once.
|
||||
- We want to keep implementation simple to avoid errors (ideally no state).
|
||||
|
||||
<!-- MarkdownTOC levels="2,3" autolink="true" indent=" " -->
|
||||
|
||||
- [Control](#control)
|
||||
- [UX](#ux)
|
||||
|
||||
<!-- /MarkdownTOC -->
|
||||
|
||||
## Control
|
||||
|
||||
In order to make this portion of the protocol simple to manage, it is defined in
|
||||
a stateless fashion. The Language Server provides two messages `file/writeBytes`
|
||||
and `file/readByteRange` with corresponding responses. We make the following
|
||||
assumptions:
|
||||
|
||||
- Each request must be completed with a response before sending another request
|
||||
for the same file.
|
||||
- Basic file information can be provided by the existing `file/info` API, which
|
||||
allows the IDE to create progress spinners and other such niceties.
|
||||
- All requests and responses are contained within the `InboundMessage` and
|
||||
`OutboundMessage` containers respectively.
|
||||
|
||||
The assumption is that, rather than encoding a stateful protocol, we instead
|
||||
rely on the IDE to control the upload and download of files by sending
|
||||
successive requests. To upload files, we intend for the IDE to use
|
||||
`file/writeBytes`, and to download files we intend for `file/readBytes` to be
|
||||
used.
|
||||
|
||||
Resumption of transfers is also handled by the IDE, which may keep track of what
|
||||
portions of a file have been written or read.
|
||||
|
||||
## UX
|
||||
|
||||
The IDE wants to be able to provide two major UX benefits to users as part of
|
||||
this work:
|
||||
|
||||
1. Loading spinners that show the progress of the file.
|
||||
2. A display of parts of the graph that are waiting on the file.
|
||||
|
||||
The first requirement here is trivially handled due to the IDE-driven nature of
|
||||
the file upload and download process. As they know the size of the file being
|
||||
transferred and get acknowledgements of each chunk of the file that is sent,
|
||||
they know both the speed and the amount of the file that has been uploaded.
|
||||
|
||||
The second requirement is being handled by the addition of the
|
||||
`Visualization.file_uploading` method to the `Visualization` portion of the
|
||||
standard library. This is a method that returns a dataflow error
|
||||
`File_Being_Uploaded path` that will flow through the graph to annotate all
|
||||
portions waiting on the file upload. All the IDE has to do is insert this
|
||||
expression implicitly into the source file while the upload is progressing, and
|
||||
it can trace the impacted nodes and display the necessary UI details.
|
@ -8,6 +8,7 @@ import Visualization_Tests.Histogram_Spec
|
||||
import Visualization_Tests.Scatter_Plot_Spec
|
||||
import Visualization_Tests.Sql_Spec
|
||||
import Visualization_Tests.Table_Spec
|
||||
import Visualization_Tests.Visualization_Spec
|
||||
|
||||
main = Test.Suite.run_main <|
|
||||
Geo_Map_Spec.spec
|
||||
@ -16,3 +17,4 @@ main = Test.Suite.run_main <|
|
||||
Scatter_Plot_Spec.spec
|
||||
Sql_Spec.spec
|
||||
Table_Spec.spec
|
||||
Visualization_Spec.spec
|
||||
|
20
test/Visualization_Tests/src/Visualization_Spec.enso
Normal file
20
test/Visualization_Tests/src/Visualization_Spec.enso
Normal file
@ -0,0 +1,20 @@
|
||||
from Standard.Base import all
|
||||
|
||||
import Standard.Examples
|
||||
import Standard.Test
|
||||
import Standard.Visualization
|
||||
|
||||
from Standard.Visualization.File_Upload import File_Being_Uploaded
|
||||
|
||||
spec = Test.group "File uploads" <|
|
||||
Test.specify "should be able to be signalled as uploading" <|
|
||||
Visualization.file_uploading "file" . should_fail_with File_Being_Uploaded
|
||||
|
||||
Test.specify "should work whether a textual or file path is provided" <|
|
||||
result_file = Visualization.file_uploading Examples.csv . catch
|
||||
result_file.file_path . should_equal Examples.csv_path
|
||||
|
||||
result_text = Visualization.file_uploading Examples.csv_path . catch
|
||||
result_text.file_path . should_equal Examples.csv_path
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user