diff --git a/CHANGELOG.md b/CHANGELOG.md index eec1dd97bcf..0cde9d209b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -640,6 +640,7 @@ dialect yet.][9435] - [Make expand_to_rows, expand_column support Rows, Tables, Column data types][9533] +- [Data Link for `Enso_File`.][9525] [debug-shortcuts]: https://github.com/enso-org/enso/blob/develop/app/gui/docs/product/shortcuts.md#debug @@ -931,6 +932,7 @@ [9406]: https://github.com/enso-org/enso/pull/9406 [9435]: https://github.com/enso-org/enso/pull/9435 [9533]: https://github.com/enso-org/enso/pull/9533 +[9525]: https://github.com/enso-org/enso/pull/9525 #### Enso Compiler diff --git a/app/ide-desktop/lib/dashboard/src/components/dashboard/DataLinkInput.tsx b/app/ide-desktop/lib/dashboard/src/components/dashboard/DataLinkInput.tsx index 307283b144e..5d23ec28f7b 100644 --- a/app/ide-desktop/lib/dashboard/src/components/dashboard/DataLinkInput.tsx +++ b/app/ide-desktop/lib/dashboard/src/components/dashboard/DataLinkInput.tsx @@ -1,9 +1,8 @@ /** @file A dynamic wizard for creating an arbitrary type of Data Link. */ import * as React from 'react' -import Ajv from 'ajv/dist/2020' - import SCHEMA from '#/data/dataLinkSchema.json' assert { type: 'json' } +import * as dataLinkValidator from '#/data/dataLinkValidator' import type * as jsonSchemaInput from '#/components/JSONSchemaInput' import JSONSchemaInput from '#/components/JSONSchemaInput' @@ -15,9 +14,6 @@ import * as error from '#/utilities/error' // ================= const DEFS: Record = SCHEMA.$defs -// eslint-disable-next-line @typescript-eslint/naming-convention -const AJV = new Ajv({ formats: { 'enso-secret': true } }) -AJV.addSchema(SCHEMA) // ==================== // === getValidator === @@ -26,7 +22,7 @@ AJV.addSchema(SCHEMA) /** Get a known schema using a path. * @throws {Error} when there is no schema present at the given path. */ function getValidator(path: string) { - return error.assert<(value: unknown) => boolean>(() => AJV.getSchema(path)) + return error.assert<(value: unknown) => boolean>(() => dataLinkValidator.AJV.getSchema(path)) } // ===================== diff --git a/app/ide-desktop/lib/dashboard/src/data/__tests__/dataLinkSchema.test.ts b/app/ide-desktop/lib/dashboard/src/data/__tests__/dataLinkSchema.test.ts index 0e6aaf3d3e0..58662ece486 100644 --- a/app/ide-desktop/lib/dashboard/src/data/__tests__/dataLinkSchema.test.ts +++ b/app/ide-desktop/lib/dashboard/src/data/__tests__/dataLinkSchema.test.ts @@ -6,12 +6,12 @@ import * as url from 'node:url' import * as v from 'vitest' -import * as validateDataLink from '#/utilities/validateDataLink' +import * as dataLinkValidator from '#/data/dataLinkValidator' v.test('correctly rejects invalid values as not matching the schema', () => { - v.expect(validateDataLink.validateDataLink({})).toBe(false) - v.expect(validateDataLink.validateDataLink('foobar')).toBe(false) - v.expect(validateDataLink.validateDataLink({ foo: 'BAR' })).toBe(false) + v.expect(dataLinkValidator.validateDataLink({})).toBe(false) + v.expect(dataLinkValidator.validateDataLink('foobar')).toBe(false) + v.expect(dataLinkValidator.validateDataLink({ foo: 'BAR' })).toBe(false) }) /** Load and parse a data-link description. */ @@ -22,7 +22,7 @@ function loadDataLinkFile(dataLinkPath: string): unknown { /** Check if the given data-link description matches the schema, reporting any errors. */ function testSchema(json: unknown, fileName: string): void { - const validate = validateDataLink.validateDataLink + const validate = dataLinkValidator.validateDataLink if (!validate(json)) { v.assert.fail(`Failed to validate ${fileName}:\n${JSON.stringify(validate.errors, null, 2)}`) } @@ -48,11 +48,19 @@ v.test('correctly validates example HTTP .datalink files with the schema', () => } }) +v.test('correctly validates example Enso_File .datalink files with the schema', () => { + const schemas = ['example-enso-file.datalink'] + for (const schema of schemas) { + const json = loadDataLinkFile(path.resolve(BASE_DATA_LINKS_ROOT, schema)) + testSchema(json, schema) + } +}) + v.test('rejects invalid schemas (Base)', () => { const invalidSchemas = ['example-http-format-invalid.datalink'] for (const schema of invalidSchemas) { const json = loadDataLinkFile(path.resolve(BASE_DATA_LINKS_ROOT, schema)) - v.expect(validateDataLink.validateDataLink(json)).toBe(false) + v.expect(dataLinkValidator.validateDataLink(json)).toBe(false) } }) diff --git a/app/ide-desktop/lib/dashboard/src/data/dataLinkSchema.json b/app/ide-desktop/lib/dashboard/src/data/dataLinkSchema.json index 1a4727e2577..3c7370c95e9 100644 --- a/app/ide-desktop/lib/dashboard/src/data/dataLinkSchema.json +++ b/app/ide-desktop/lib/dashboard/src/data/dataLinkSchema.json @@ -5,6 +5,7 @@ "title": "Data Link", "anyOf": [ { "$ref": "#/$defs/S3DataLink" }, + { "$ref": "#/$defs/EnsoFileDataLink" }, { "$ref": "#/$defs/HttpFetchDataLink" }, { "$ref": "#/$defs/PostgresDataLink" }, { "$ref": "#/$defs/SnowflakeDataLink" } @@ -110,6 +111,23 @@ }, "required": ["type", "libraryName", "uri", "auth"] }, + "EnsoFileDataLink": { + "title": "Enso File", + "type": "object", + "properties": { + "type": { "title": "Type", "const": "Enso_File", "type": "string" }, + "libraryName": { "const": "Standard.Base" }, + "path": { + "title": "Path", + "description": "Must start with \"enso:///\".", + "type": "string", + "pattern": "^enso://.+/.*$", + "format": "enso-file" + }, + "format": { "title": "Format", "$ref": "#/$defs/Format" } + }, + "required": ["type", "libraryName", "path"] + }, "HttpFetchDataLink": { "$comment": "missing and ", "title": "HTTP Fetch", diff --git a/app/ide-desktop/lib/dashboard/src/utilities/validateDataLink.ts b/app/ide-desktop/lib/dashboard/src/data/dataLinkValidator.ts similarity index 78% rename from app/ide-desktop/lib/dashboard/src/utilities/validateDataLink.ts rename to app/ide-desktop/lib/dashboard/src/data/dataLinkValidator.ts index af4e475637c..615a7d0d540 100644 --- a/app/ide-desktop/lib/dashboard/src/utilities/validateDataLink.ts +++ b/app/ide-desktop/lib/dashboard/src/data/dataLinkValidator.ts @@ -1,4 +1,4 @@ -/** @file Validation functions related to Data Links. */ +/** @file AJV instance configured for data links. */ import type * as ajv from 'ajv/dist/2020' import Ajv from 'ajv/dist/2020' @@ -7,8 +7,9 @@ import SCHEMA from '#/data/dataLinkSchema.json' assert { type: 'json' } import * as error from '#/utilities/error' // eslint-disable-next-line @typescript-eslint/naming-convention -const AJV = new Ajv({ formats: { 'enso-secret': true } }) +export const AJV = new Ajv({ formats: { 'enso-secret': true, 'enso-file': true } }) AJV.addSchema(SCHEMA) + // This is a function, even though it does not contain function syntax. // eslint-disable-next-line no-restricted-syntax export const validateDataLink = error.assert(() => diff --git a/app/ide-desktop/lib/dashboard/src/index.html b/app/ide-desktop/lib/dashboard/src/index.html index 84f7c3f2bb3..9845828fa65 120000 --- a/app/ide-desktop/lib/dashboard/src/index.html +++ b/app/ide-desktop/lib/dashboard/src/index.html @@ -1 +1 @@ -../../content/src/index.html \ No newline at end of file +../../content/src/index.html diff --git a/app/ide-desktop/lib/dashboard/src/layouts/AssetProperties.tsx b/app/ide-desktop/lib/dashboard/src/layouts/AssetProperties.tsx index 803d45f91f4..d411617fa01 100644 --- a/app/ide-desktop/lib/dashboard/src/layouts/AssetProperties.tsx +++ b/app/ide-desktop/lib/dashboard/src/layouts/AssetProperties.tsx @@ -3,6 +3,8 @@ import * as React from 'react' import PenIcon from 'enso-assets/pen.svg' +import * as dataLinkValidator from '#/data/dataLinkValidator' + import * as toastAndLogHooks from '#/hooks/toastAndLogHooks' import * as authProvider from '#/providers/AuthProvider' @@ -25,7 +27,6 @@ import type AssetQuery from '#/utilities/AssetQuery' import type AssetTreeNode from '#/utilities/AssetTreeNode' import * as object from '#/utilities/object' import * as permissions from '#/utilities/permissions' -import * as validateDataLink from '#/utilities/validateDataLink' // ======================= // === AssetProperties === @@ -60,7 +61,7 @@ export default function AssetProperties(props: AssetPropertiesProps) { ) const [isDataLinkFetched, setIsDataLinkFetched] = React.useState(false) const isDataLinkSubmittable = React.useMemo( - () => validateDataLink.validateDataLink(dataLinkValue), + () => dataLinkValidator.validateDataLink(dataLinkValue), [dataLinkValue] ) const setItem = React.useCallback( diff --git a/app/ide-desktop/lib/dashboard/src/modals/UpsertDataLinkModal.tsx b/app/ide-desktop/lib/dashboard/src/modals/UpsertDataLinkModal.tsx index 75bd42e1352..d667da3d8c3 100644 --- a/app/ide-desktop/lib/dashboard/src/modals/UpsertDataLinkModal.tsx +++ b/app/ide-desktop/lib/dashboard/src/modals/UpsertDataLinkModal.tsx @@ -2,6 +2,7 @@ import * as React from 'react' import SCHEMA from '#/data/dataLinkSchema.json' assert { type: 'json' } +import * as dataLinkValidator from '#/data/dataLinkValidator' import * as modalProvider from '#/providers/ModalProvider' import * as textProvider from '#/providers/TextProvider' @@ -10,7 +11,6 @@ import DataLinkInput from '#/components/dashboard/DataLinkInput' import Modal from '#/components/Modal' import * as jsonSchema from '#/utilities/jsonSchema' -import * as validateDataLink from '#/utilities/validateDataLink' // ================= // === Constants === @@ -36,7 +36,7 @@ export default function UpsertDataLinkModal(props: UpsertDataLinkModalProps) { const { getText } = textProvider.useText() const [name, setName] = React.useState('') const [value, setValue] = React.useState | null>(INITIAL_DATA_LINK_VALUE) - const isValueSubmittable = React.useMemo(() => validateDataLink.validateDataLink(value), [value]) + const isValueSubmittable = React.useMemo(() => dataLinkValidator.validateDataLink(value), [value]) const isSubmittable = name !== '' && isValueSubmittable return ( diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/S3/S3_Data_Link.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/S3/S3_Data_Link.enso index a878fa0966e..597f07a37c4 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/S3/S3_Data_Link.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/S3/S3_Data_Link.enso @@ -1,3 +1,5 @@ +private + from Standard.Base import all import Standard.Base.Errors.Illegal_State.Illegal_State import Standard.Base.System.Input_Stream.Input_Stream diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/src/S3/S3_File.enso b/distribution/lib/Standard/AWS/0.0.0-dev/src/S3/S3_File.enso index 821efd9347e..8dfd74da485 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/src/S3/S3_File.enso +++ b/distribution/lib/Standard/AWS/0.0.0-dev/src/S3/S3_File.enso @@ -126,7 +126,7 @@ type S3_File with_input_stream self (open_options : Vector) action = if self.is_directory then Error.throw (Illegal_Argument.Error "S3 folders cannot be opened as a stream." self.uri) else open_as_data_link = (open_options.contains Data_Link_Access.No_Follow . not) && (Data_Link.is_data_link self) if open_as_data_link then Data_Link.read_data_link_as_stream self open_options action else - File_Access.ensure_only_allowed_options "with_output_stream" [File_Access.Read, Data_Link_Access.No_Follow] open_options <| + File_Access.ensure_only_allowed_options "with_input_stream" [File_Access.Read, Data_Link_Access.No_Follow] open_options <| response_body = translate_file_errors self <| S3.get_object self.s3_path.bucket self.s3_path.key self.credentials delimiter=S3_Path.delimiter response_body.with_stream action diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Enso_File_Data_Link.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Enso_File_Data_Link.enso new file mode 100644 index 00000000000..045db8294e1 --- /dev/null +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Enso_File_Data_Link.enso @@ -0,0 +1,40 @@ +private + +import project.Any.Any +import project.Data.Text.Text +import project.Data.Vector.Vector +import project.Enso_Cloud.Enso_File.Enso_File +import project.Errors.Problem_Behavior.Problem_Behavior +import project.Nothing.Nothing +import project.System.File_Format.Auto_Detect +import project.System.Input_Stream.Input_Stream +from project.Enso_Cloud.Data_Link import Data_Link_With_Input_Stream, parse_format +from project.Enso_Cloud.Public_Utils import get_optional_field, get_required_field + + +## PRIVATE +type Enso_File_Data_Link + ## PRIVATE + Value (path : Text) format_json + + ## PRIVATE + parse json -> Enso_File_Data_Link = + path = get_required_field "path" json expected_type=Text + format_json = get_optional_field "format" json + Enso_File_Data_Link.Value path format_json + + ## PRIVATE + read self (format = Auto_Detect) (on_problems : Problem_Behavior) = + effective_format = if format != Auto_Detect then format else parse_format self.format_json + self.as_file.read effective_format on_problems + + ## PRIVATE + as_file self -> Enso_File = + Enso_File.new self.path + + ## PRIVATE + with_input_stream self (open_options : Vector) (action : Input_Stream -> Any) -> Any = + self.as_file.with_input_stream open_options action + +## PRIVATE +Data_Link_With_Input_Stream.from (that:Enso_File_Data_Link) = Data_Link_With_Input_Stream.Value that diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Fetch_Data_Link.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Internal/HTTP_Fetch_Data_Link.enso similarity index 87% rename from distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Fetch_Data_Link.enso rename to distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Internal/HTTP_Fetch_Data_Link.enso index b59401ed85c..bdc7e088ac9 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/HTTP_Fetch_Data_Link.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Internal/HTTP_Fetch_Data_Link.enso @@ -1,3 +1,5 @@ +private + import project.Any.Any import project.Data.Text.Text import project.Data.Vector.Vector @@ -5,12 +7,9 @@ import project.Errors.Problem_Behavior.Problem_Behavior import project.Network.HTTP.HTTP import project.Network.HTTP.HTTP_Method.HTTP_Method import project.Network.HTTP.Request.Request -import project.Nothing.Nothing -import project.System.File.Data_Link_Access.Data_Link_Access import project.System.File.File_Access.File_Access import project.System.File_Format.Auto_Detect import project.System.Input_Stream.Input_Stream -from project.Data.Boolean import Boolean, False, True from project.Enso_Cloud.Data_Link import Data_Link_With_Input_Stream, parse_format, parse_secure_value from project.Enso_Cloud.Public_Utils import get_optional_field, get_required_field @@ -38,7 +37,7 @@ type HTTP_Fetch_Data_Link ## PRIVATE with_input_stream self (open_options : Vector) (action : Input_Stream -> Any) -> Any = - File_Access.ensure_only_allowed_options "with_output_stream" [File_Access.Read] open_options <| + File_Access.ensure_only_allowed_options "with_input_stream" [File_Access.Read] open_options <| response = HTTP.new.request self.request response.body.with_stream action diff --git a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Data_Link/Postgres_Data_Link.enso b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Data_Link/Postgres_Data_Link.enso index a0ff69a9ce9..aad697c576d 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Data_Link/Postgres_Data_Link.enso +++ b/distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Data_Link/Postgres_Data_Link.enso @@ -1,3 +1,5 @@ +private + from Standard.Base import all import Standard.Base.Errors.Illegal_Argument.Illegal_Argument import Standard.Base.System.Input_Stream.Input_Stream diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoFileDataLinkSPI.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoFileDataLinkSPI.java new file mode 100644 index 00000000000..6e496d4c61f --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoFileDataLinkSPI.java @@ -0,0 +1,19 @@ +package org.enso.base.enso_cloud; + +@org.openide.util.lookup.ServiceProvider(service = DataLinkSPI.class) +public class EnsoFileDataLinkSPI extends DataLinkSPI { + @Override + protected String getModuleName() { + return "Standard.Base.Enso_Cloud.Internal.Enso_File_Data_Link"; + } + + @Override + protected String getTypeName() { + return "Enso_File_Data_Link"; + } + + @Override + protected String getLinkTypeName() { + return "Enso_File"; + } +} diff --git a/std-bits/base/src/main/java/org/enso/base/net/http/HTTPFetchDataLinkSPI.java b/std-bits/base/src/main/java/org/enso/base/net/http/HTTPFetchDataLinkSPI.java index f8d98e016f0..027b157de63 100644 --- a/std-bits/base/src/main/java/org/enso/base/net/http/HTTPFetchDataLinkSPI.java +++ b/std-bits/base/src/main/java/org/enso/base/net/http/HTTPFetchDataLinkSPI.java @@ -6,7 +6,7 @@ import org.enso.base.enso_cloud.DataLinkSPI; public class HTTPFetchDataLinkSPI extends DataLinkSPI { @Override protected String getModuleName() { - return "Standard.Base.Network.HTTP.HTTP_Fetch_Data_Link"; + return "Standard.Base.Network.HTTP.Internal.HTTP_Fetch_Data_Link"; } @Override diff --git a/test/Base_Tests/data/datalinks/example-enso-file.datalink b/test/Base_Tests/data/datalinks/example-enso-file.datalink new file mode 100644 index 00000000000..b1354830d88 --- /dev/null +++ b/test/Base_Tests/data/datalinks/example-enso-file.datalink @@ -0,0 +1,5 @@ +{ + "type": "Enso_File", + "libraryName": "Standard.Base", + "path": "enso://PLACEHOLDER_ORG_NAME/test-directory/another.txt" +} diff --git a/test/Base_Tests/src/Network/Enso_Cloud/Cloud_Data_Link_Spec.enso b/test/Base_Tests/src/Network/Enso_Cloud/Cloud_Data_Link_Spec.enso index ba3a8353d8f..7b5d61e9949 100644 --- a/test/Base_Tests/src/Network/Enso_Cloud/Cloud_Data_Link_Spec.enso +++ b/test/Base_Tests/src/Network/Enso_Cloud/Cloud_Data_Link_Spec.enso @@ -44,8 +44,27 @@ add_specs suite_builder setup:Cloud_Tests_Setup = setup.with_prepared_environmen Test.expect_panic Illegal_Argument <| Data_Link_Format.read_raw_config temp_file + group_builder.specify "should be able to read a local datalink to an Enso File" <| + datalink = replace_org_name_in_data_link (enso_project.data / "datalinks" / "example-enso-file.datalink") + datalink.read . should_equal "Hello Another!" + + group_builder.specify "should be able to read a datalink in the Cloud to Enso File" <| + # TODO currently this link is created manually, later we should be generating it as part of the test + datalink = Enso_File.new "enso://"+Enso_User.current.name+"/TestDataLink-EnsoFile" + datalink.read . should_equal "Hello Another!" + + main filter=Nothing = setup = Cloud_Tests_Setup.prepare suite = Test.build suite_builder-> add_specs suite_builder setup suite.run_with_filter filter + + +## Reads the datalink as plain text and replaces the placeholder organization name. +replace_org_name_in_data_link base_file = + content = Data_Link_Format.read_raw_config base_file + org_name = Enso_User.current.name + new_content = content.replace "PLACEHOLDER_ORG_NAME" org_name + temp_file = File.create_temporary_file prefix=base_file.name suffix=base_file.extension + Data_Link_Format.write_raw_config temp_file new_content replace_existing=True . if_not_error temp_file