Data Link for reading Enso_File (#9525)

- Closes #9282

CHANGELOG.md

@@ -640,6 +640,7 @@
   dialect yet.][9435]
 - [Make expand_to_rows, expand_column support Rows, Tables, Column data
   types][9533]
+- [Data Link for `Enso_File`.][9525]
 
 [debug-shortcuts]:
   https://github.com/enso-org/enso/blob/develop/app/gui/docs/product/shortcuts.md#debug
@@ -931,6 +932,7 @@
 [9406]: https://github.com/enso-org/enso/pull/9406
 [9435]: https://github.com/enso-org/enso/pull/9435
 [9533]: https://github.com/enso-org/enso/pull/9533
+[9525]: https://github.com/enso-org/enso/pull/9525
 
 #### Enso Compiler
 

app/ide-desktop/lib/dashboard/src/components/dashboard/DataLinkInput.tsx

@@ -1,9 +1,8 @@
 /** @file A dynamic wizard for creating an arbitrary type of Data Link. */
 import * as React from 'react'
 
-import Ajv from 'ajv/dist/2020'
-
 import SCHEMA from '#/data/dataLinkSchema.json' assert { type: 'json' }
+import * as dataLinkValidator from '#/data/dataLinkValidator'
 
 import type * as jsonSchemaInput from '#/components/JSONSchemaInput'
 import JSONSchemaInput from '#/components/JSONSchemaInput'
@@ -15,9 +14,6 @@ import * as error from '#/utilities/error'
 // =================
 
 const DEFS: Record<string, object> = SCHEMA.$defs
-// eslint-disable-next-line @typescript-eslint/naming-convention
-const AJV = new Ajv({ formats: { 'enso-secret': true } })
-AJV.addSchema(SCHEMA)
 
 // ====================
 // === getValidator ===
@@ -26,7 +22,7 @@ AJV.addSchema(SCHEMA)
 /** Get a known schema using a path.
  * @throws {Error} when there is no schema present at the given path. */
 function getValidator(path: string) {
-  return error.assert<(value: unknown) => boolean>(() => AJV.getSchema(path))
+  return error.assert<(value: unknown) => boolean>(() => dataLinkValidator.AJV.getSchema(path))
 }
 
 // =====================

app/ide-desktop/lib/dashboard/src/data/__tests__/dataLinkSchema.test.ts

@@ -6,12 +6,12 @@ import * as url from 'node:url'
 
 import * as v from 'vitest'
 
-import * as validateDataLink from '#/utilities/validateDataLink'
+import * as dataLinkValidator from '#/data/dataLinkValidator'
 
 v.test('correctly rejects invalid values as not matching the schema', () => {
-  v.expect(validateDataLink.validateDataLink({})).toBe(false)
-  v.expect(validateDataLink.validateDataLink('foobar')).toBe(false)
-  v.expect(validateDataLink.validateDataLink({ foo: 'BAR' })).toBe(false)
+  v.expect(dataLinkValidator.validateDataLink({})).toBe(false)
+  v.expect(dataLinkValidator.validateDataLink('foobar')).toBe(false)
+  v.expect(dataLinkValidator.validateDataLink({ foo: 'BAR' })).toBe(false)
 })
 
 /** Load and parse a data-link description. */
@@ -22,7 +22,7 @@ function loadDataLinkFile(dataLinkPath: string): unknown {
 
 /** Check if the given data-link description matches the schema, reporting any errors. */
 function testSchema(json: unknown, fileName: string): void {
-  const validate = validateDataLink.validateDataLink
+  const validate = dataLinkValidator.validateDataLink
   if (!validate(json)) {
     v.assert.fail(`Failed to validate ${fileName}:\n${JSON.stringify(validate.errors, null, 2)}`)
   }
@@ -48,11 +48,19 @@ v.test('correctly validates example HTTP .datalink files with the schema', () =>
   }
 })
 
+v.test('correctly validates example Enso_File .datalink files with the schema', () => {
+  const schemas = ['example-enso-file.datalink']
+  for (const schema of schemas) {
+    const json = loadDataLinkFile(path.resolve(BASE_DATA_LINKS_ROOT, schema))
+    testSchema(json, schema)
+  }
+})
+
 v.test('rejects invalid schemas (Base)', () => {
   const invalidSchemas = ['example-http-format-invalid.datalink']
   for (const schema of invalidSchemas) {
     const json = loadDataLinkFile(path.resolve(BASE_DATA_LINKS_ROOT, schema))
-    v.expect(validateDataLink.validateDataLink(json)).toBe(false)
+    v.expect(dataLinkValidator.validateDataLink(json)).toBe(false)
   }
 })
 

app/ide-desktop/lib/dashboard/src/data/dataLinkSchema.json

@@ -5,6 +5,7 @@
       "title": "Data Link",
       "anyOf": [
         { "$ref": "#/$defs/S3DataLink" },
+        { "$ref": "#/$defs/EnsoFileDataLink" },
         { "$ref": "#/$defs/HttpFetchDataLink" },
         { "$ref": "#/$defs/PostgresDataLink" },
         { "$ref": "#/$defs/SnowflakeDataLink" }
@@ -110,6 +111,23 @@
       },
       "required": ["type", "libraryName", "uri", "auth"]
     },
+    "EnsoFileDataLink": {
+      "title": "Enso File",
+      "type": "object",
+      "properties": {
+        "type": { "title": "Type", "const": "Enso_File", "type": "string" },
+        "libraryName": { "const": "Standard.Base" },
+        "path": {
+          "title": "Path",
+          "description": "Must start with \"enso://<organization-name>/\".",
+          "type": "string",
+          "pattern": "^enso://.+/.*$",
+          "format": "enso-file"
+        },
+        "format": { "title": "Format", "$ref": "#/$defs/Format" }
+      },
+      "required": ["type", "libraryName", "path"]
+    },
     "HttpFetchDataLink": {
       "$comment": "missing <headers with secrets> and <query string with secrets>",
       "title": "HTTP Fetch",

app/ide-desktop/lib/dashboard/src/data/dataLinkValidator.ts

@@ -1,4 +1,4 @@
-/** @file Validation functions related to Data Links. */
+/** @file AJV instance configured for data links. */
 import type * as ajv from 'ajv/dist/2020'
 import Ajv from 'ajv/dist/2020'
 
@@ -7,8 +7,9 @@ import SCHEMA from '#/data/dataLinkSchema.json' assert { type: 'json' }
 import * as error from '#/utilities/error'
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
-const AJV = new Ajv({ formats: { 'enso-secret': true } })
+export const AJV = new Ajv({ formats: { 'enso-secret': true, 'enso-file': true } })
 AJV.addSchema(SCHEMA)
+
 // This is a function, even though it does not contain function syntax.
 // eslint-disable-next-line no-restricted-syntax
 export const validateDataLink = error.assert<ajv.ValidateFunction>(() =>

app/ide-desktop/lib/dashboard/src/layouts/AssetProperties.tsx

@@ -3,6 +3,8 @@ import * as React from 'react'
 
 import PenIcon from 'enso-assets/pen.svg'
 
+import * as dataLinkValidator from '#/data/dataLinkValidator'
+
 import * as toastAndLogHooks from '#/hooks/toastAndLogHooks'
 
 import * as authProvider from '#/providers/AuthProvider'
@@ -25,7 +27,6 @@ import type AssetQuery from '#/utilities/AssetQuery'
 import type AssetTreeNode from '#/utilities/AssetTreeNode'
 import * as object from '#/utilities/object'
 import * as permissions from '#/utilities/permissions'
-import * as validateDataLink from '#/utilities/validateDataLink'
 
 // =======================
 // === AssetProperties ===
@@ -60,7 +61,7 @@ export default function AssetProperties(props: AssetPropertiesProps) {
   )
   const [isDataLinkFetched, setIsDataLinkFetched] = React.useState(false)
   const isDataLinkSubmittable = React.useMemo(
-    () => validateDataLink.validateDataLink(dataLinkValue),
+    () => dataLinkValidator.validateDataLink(dataLinkValue),
     [dataLinkValue]
   )
   const setItem = React.useCallback(

app/ide-desktop/lib/dashboard/src/modals/UpsertDataLinkModal.tsx

@@ -2,6 +2,7 @@
 import * as React from 'react'
 
 import SCHEMA from '#/data/dataLinkSchema.json' assert { type: 'json' }
+import * as dataLinkValidator from '#/data/dataLinkValidator'
 
 import * as modalProvider from '#/providers/ModalProvider'
 import * as textProvider from '#/providers/TextProvider'
@@ -10,7 +11,6 @@ import DataLinkInput from '#/components/dashboard/DataLinkInput'
 import Modal from '#/components/Modal'
 
 import * as jsonSchema from '#/utilities/jsonSchema'
-import * as validateDataLink from '#/utilities/validateDataLink'
 
 // =================
 // === Constants ===
@@ -36,7 +36,7 @@ export default function UpsertDataLinkModal(props: UpsertDataLinkModalProps) {
   const { getText } = textProvider.useText()
   const [name, setName] = React.useState('')
   const [value, setValue] = React.useState<NonNullable<unknown> | null>(INITIAL_DATA_LINK_VALUE)
-  const isValueSubmittable = React.useMemo(() => validateDataLink.validateDataLink(value), [value])
+  const isValueSubmittable = React.useMemo(() => dataLinkValidator.validateDataLink(value), [value])
   const isSubmittable = name !== '' && isValueSubmittable
 
   return (

distribution/lib/Standard/AWS/0.0.0-dev/src/S3/S3_Data_Link.enso

@@ -1,3 +1,5 @@
+private
+
 from Standard.Base import all
 import Standard.Base.Errors.Illegal_State.Illegal_State
 import Standard.Base.System.Input_Stream.Input_Stream

distribution/lib/Standard/AWS/0.0.0-dev/src/S3/S3_File.enso

@@ -126,7 +126,7 @@ type S3_File
     with_input_stream self (open_options : Vector) action = if self.is_directory then Error.throw (Illegal_Argument.Error "S3 folders cannot be opened as a stream." self.uri) else
         open_as_data_link = (open_options.contains Data_Link_Access.No_Follow . not) && (Data_Link.is_data_link self)
         if open_as_data_link then Data_Link.read_data_link_as_stream self open_options action else
-            File_Access.ensure_only_allowed_options "with_output_stream" [File_Access.Read, Data_Link_Access.No_Follow] open_options <|
+            File_Access.ensure_only_allowed_options "with_input_stream" [File_Access.Read, Data_Link_Access.No_Follow] open_options <|
                 response_body = translate_file_errors self <| S3.get_object self.s3_path.bucket self.s3_path.key self.credentials delimiter=S3_Path.delimiter
                 response_body.with_stream action
 

distribution/lib/Standard/Base/0.0.0-dev/src/Enso_Cloud/Internal/Enso_File_Data_Link.enso

@@ -0,0 +1,40 @@
+private
+
+import project.Any.Any
+import project.Data.Text.Text
+import project.Data.Vector.Vector
+import project.Enso_Cloud.Enso_File.Enso_File
+import project.Errors.Problem_Behavior.Problem_Behavior
+import project.Nothing.Nothing
+import project.System.File_Format.Auto_Detect
+import project.System.Input_Stream.Input_Stream
+from project.Enso_Cloud.Data_Link import Data_Link_With_Input_Stream, parse_format
+from project.Enso_Cloud.Public_Utils import get_optional_field, get_required_field
+
+
+## PRIVATE
+type Enso_File_Data_Link
+    ## PRIVATE
+    Value (path : Text) format_json
+
+    ## PRIVATE
+    parse json -> Enso_File_Data_Link =
+        path = get_required_field "path" json expected_type=Text
+        format_json = get_optional_field "format" json
+        Enso_File_Data_Link.Value path format_json
+
+    ## PRIVATE
+    read self (format = Auto_Detect) (on_problems : Problem_Behavior) =
+        effective_format = if format != Auto_Detect then format else parse_format self.format_json
+        self.as_file.read effective_format on_problems
+
+    ## PRIVATE
+    as_file self -> Enso_File =
+        Enso_File.new self.path
+
+    ## PRIVATE
+    with_input_stream self (open_options : Vector) (action : Input_Stream -> Any) -> Any =
+        self.as_file.with_input_stream open_options action
+
+## PRIVATE
+Data_Link_With_Input_Stream.from (that:Enso_File_Data_Link) = Data_Link_With_Input_Stream.Value that

distribution/lib/Standard/Base/0.0.0-dev/src/Network/HTTP/Internal/HTTP_Fetch_Data_Link.enso

@@ -1,3 +1,5 @@
+private
+
 import project.Any.Any
 import project.Data.Text.Text
 import project.Data.Vector.Vector
@@ -5,12 +7,9 @@ import project.Errors.Problem_Behavior.Problem_Behavior
 import project.Network.HTTP.HTTP
 import project.Network.HTTP.HTTP_Method.HTTP_Method
 import project.Network.HTTP.Request.Request
-import project.Nothing.Nothing
-import project.System.File.Data_Link_Access.Data_Link_Access
 import project.System.File.File_Access.File_Access
 import project.System.File_Format.Auto_Detect
 import project.System.Input_Stream.Input_Stream
-from project.Data.Boolean import Boolean, False, True
 from project.Enso_Cloud.Data_Link import Data_Link_With_Input_Stream, parse_format, parse_secure_value
 from project.Enso_Cloud.Public_Utils import get_optional_field, get_required_field
 
@@ -38,7 +37,7 @@ type HTTP_Fetch_Data_Link
 
     ## PRIVATE
     with_input_stream self (open_options : Vector) (action : Input_Stream -> Any) -> Any =
-        File_Access.ensure_only_allowed_options "with_output_stream" [File_Access.Read] open_options <|
+        File_Access.ensure_only_allowed_options "with_input_stream" [File_Access.Read] open_options <|
             response = HTTP.new.request self.request
             response.body.with_stream action
 

distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Data_Link/Postgres_Data_Link.enso

@@ -1,3 +1,5 @@
+private
+
 from Standard.Base import all
 import Standard.Base.Errors.Illegal_Argument.Illegal_Argument
 import Standard.Base.System.Input_Stream.Input_Stream

std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoFileDataLinkSPI.java

@@ -0,0 +1,19 @@
+package org.enso.base.enso_cloud;
+
+@org.openide.util.lookup.ServiceProvider(service = DataLinkSPI.class)
+public class EnsoFileDataLinkSPI extends DataLinkSPI {
+  @Override
+  protected String getModuleName() {
+    return "Standard.Base.Enso_Cloud.Internal.Enso_File_Data_Link";
+  }
+
+  @Override
+  protected String getTypeName() {
+    return "Enso_File_Data_Link";
+  }
+
+  @Override
+  protected String getLinkTypeName() {
+    return "Enso_File";
+  }
+}

std-bits/base/src/main/java/org/enso/base/net/http/HTTPFetchDataLinkSPI.java

@@ -6,7 +6,7 @@ import org.enso.base.enso_cloud.DataLinkSPI;
 public class HTTPFetchDataLinkSPI extends DataLinkSPI {
   @Override
   protected String getModuleName() {
-    return "Standard.Base.Network.HTTP.HTTP_Fetch_Data_Link";
+    return "Standard.Base.Network.HTTP.Internal.HTTP_Fetch_Data_Link";
   }
 
   @Override

test/Base_Tests/data/datalinks/example-enso-file.datalink

@@ -0,0 +1,5 @@
+{
+    "type": "Enso_File",
+    "libraryName": "Standard.Base",
+    "path": "enso://PLACEHOLDER_ORG_NAME/test-directory/another.txt"
+}

test/Base_Tests/src/Network/Enso_Cloud/Cloud_Data_Link_Spec.enso

@@ -44,8 +44,27 @@ add_specs suite_builder setup:Cloud_Tests_Setup = setup.with_prepared_environmen
             Test.expect_panic Illegal_Argument <|
                 Data_Link_Format.read_raw_config temp_file
 
+        group_builder.specify "should be able to read a local datalink to an Enso File" <|
+            datalink = replace_org_name_in_data_link (enso_project.data / "datalinks" / "example-enso-file.datalink")
+            datalink.read . should_equal "Hello Another!"
+
+        group_builder.specify "should be able to read a datalink in the Cloud to Enso File" <|
+            # TODO currently this link is created manually, later we should be generating it as part of the test
+            datalink = Enso_File.new "enso://"+Enso_User.current.name+"/TestDataLink-EnsoFile"
+            datalink.read . should_equal "Hello Another!"
+
+
 main filter=Nothing =
     setup = Cloud_Tests_Setup.prepare
     suite = Test.build suite_builder->
         add_specs suite_builder setup
     suite.run_with_filter filter
+
+
+## Reads the datalink as plain text and replaces the placeholder organization name.
+replace_org_name_in_data_link base_file =
+    content = Data_Link_Format.read_raw_config base_file
+    org_name = Enso_User.current.name
+    new_content = content.replace "PLACEHOLDER_ORG_NAME" org_name
+    temp_file = File.create_temporary_file prefix=base_file.name suffix=base_file.extension
+    Data_Link_Format.write_raw_config temp_file new_content replace_existing=True . if_not_error temp_file