Allow secrets in AWS_Credential (#8774)

- Closes #8722

distribution/lib/Standard/AWS/0.0.0-dev/src/AWS_Credential.enso

@@ -1,7 +1,9 @@
 from Standard.Base import all
 from Standard.Base.Metadata import make_single_choice, Widget
+import Standard.Base.Data.Enso_Cloud.Utils as Cloud_Utils
 
 polyglot java import org.enso.aws.ProfileReader
+polyglot java import org.enso.aws.AwsCredential
 
 type AWS_Credential
     ## Access using IAM via an AWS profile.
@@ -16,7 +18,7 @@ type AWS_Credential
        Arguments:
        - access_key_id: AWS access key ID.
        - secret_access_key: AWS secret access key.
-    Key access_key_id:Text secret_access_key:Text
+    Key access_key_id:Text|Enso_Secret secret_access_key:Text|Enso_Secret
 
     ## Get a list of the available profiles
     profile_names : Vector Text
@@ -28,3 +30,11 @@ type AWS_Credential
     default_widget =
         fqn = Meta.get_qualified_type_name AWS_Credential
         make_single_choice [["default", "Nothing"], ["by profile", fqn + ".Profile"], ["by key", fqn + ".Key"]]
+
+    ## PRIVATE
+    as_java : AWS_Credential|Nothing -> AwsCredential
+    as_java (credential : AWS_Credential | Nothing) = case credential of
+        AWS_Credential.Profile profile -> AwsCredential.Profile.new profile
+        AWS_Credential.Key access_key_id secret_access_key ->
+            AwsCredential.Key.new (Cloud_Utils.as_hideable_value access_key_id) (Cloud_Utils.as_hideable_value secret_access_key)
+        Nothing -> AwsCredential.Default.new

distribution/lib/Standard/AWS/0.0.0-dev/src/Internal/Auth.enso

@@ -1,21 +0,0 @@
-from Standard.Base import all
-
-import project.AWS_Credential.AWS_Credential
-
-polyglot java import software.amazon.awssdk.auth.credentials.AwsBasicCredentials
-polyglot java import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider
-polyglot java import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider
-polyglot java import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider
-polyglot java import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider
-
-## PRIVATE
-   Create the Java credentialsProvider object.
-create_provider : (AWS_Credential | Nothing) -> AwsCredentialsProvider
-create_provider (credentials : AWS_Credential | Nothing) = case credentials of
-    AWS_Credential.Profile p ->
-        ProfileCredentialsProvider.builder.profileName p . build
-    AWS_Credential.Key key secret ->
-        credentials = AwsBasicCredentials.create key secret
-        StaticCredentialsProvider.create credentials
-    Nothing ->
-        DefaultCredentialsProvider.builder.build

distribution/lib/Standard/AWS/0.0.0-dev/src/S3/S3.enso

@@ -10,9 +10,9 @@ import project.Errors.AWS_SDK_Error
 import project.Errors.More_Records_Available
 import project.Errors.S3_Bucket_Not_Found
 import project.Errors.S3_Error
-import project.Internal.Auth
 
 polyglot java import java.io.IOException
+polyglot java import org.enso.aws.ClientBuilder
 polyglot java import software.amazon.awssdk.core.exception.SdkClientException
 polyglot java import software.amazon.awssdk.services.s3.model.GetObjectRequest
 polyglot java import software.amazon.awssdk.services.s3.model.HeadBucketRequest
@@ -151,8 +151,8 @@ handle_s3_errors ~action bucket="" key="" =
 ## PRIVATE
 make_client : (AWS_Credential | Nothing) -> S3Client
 make_client credentials:(AWS_Credential | Nothing) =
-    provider = Auth.create_provider credentials
-    S3Client.builder.credentialsProvider provider . build
+    builder = ClientBuilder.new (AWS_Credential.as_java credentials)
+    builder.buildS3Client
 
 ## PRIVATE
    Utility method for running an action with Java exceptions mapping.

engine/language-server/src/test/scala/org/enso/languageserver/websocket/json/LibrariesTest.scala

@@ -21,13 +21,17 @@ import org.enso.librarymanager.published.repository.{
 }
 import org.enso.logger.ReportLogsOnFailure
 import org.enso.pkg.{Config, Contact, Package, PackageManager}
+import org.enso.testkit.FlakySpec
 import org.enso.yaml.YamlHelper
 
 import java.nio.file.Files
 import java.nio.file.Path
 import scala.concurrent.duration._
 
-class LibrariesTest extends BaseServerTest with ReportLogsOnFailure {
+class LibrariesTest
+    extends BaseServerTest
+    with ReportLogsOnFailure
+    with FlakySpec {
   private val libraryRepositoryPort: Int = 47308
 
   private val exampleRepo = new ExampleRepository(
@@ -55,7 +59,7 @@ class LibrariesTest extends BaseServerTest with ReportLogsOnFailure {
   )
 
   "LocalLibraryManager" should {
-    "create a library project and include it on the list of local projects" in {
+    "create a library project and include it on the list of local projects" taggedAs Flaky in {
       val client          = getInitialisedWsClient()
       val testLibraryName = LibraryName("user", "My_Local_Lib")
 

lib/scala/json-rpc-server-test/src/main/scala/org/enso/jsonrpc/test/JsonRpcServerTestKit.scala

@@ -153,7 +153,7 @@ abstract class JsonRpcServerTestKit
     }
 
     def expectSomeJson(
-      timeout: FiniteDuration = 5.seconds.dilated
+      timeout: FiniteDuration = 10.seconds.dilated
     )(implicit pos: Position): Json = {
       val parsed = parse(expectMessage(timeout))
       inside(parsed) { case Right(json) => json }

lib/scala/library-manager-test/src/main/scala/org/enso/librarymanager/published/repository/DummyRepository.scala

@@ -181,7 +181,7 @@ abstract class DummyRepository(toolsRootDirectory: Path) {
         .toAbsolutePath
         .normalize
 
-    // We can ommit installation step on CI because there is a separate step
+    // We can omit installation step on CI because there is a separate step
     // executing `npm install` command before the tests.
     if (!DummyRepository.isCI) {
       val preinstallCommand =
@@ -197,7 +197,7 @@ abstract class DummyRepository(toolsRootDirectory: Path) {
       if (preinstallExitCode != 0)
         throw new RuntimeException(
           s"Failed to preinstall the Library Repository Server dependencies: " +
-          s"npm exited with code $preinstallCommand."
+          s"npm ($preinstallCommand) exited with code $preinstallExitCode."
         )
     }
 

std-bits/aws/src/main/java/org/enso/aws/AwsCredential.java

@@ -0,0 +1,11 @@
+package org.enso.aws;
+
+import org.enso.base.enso_cloud.HideableValue;
+
+public sealed interface AwsCredential {
+  record Key(HideableValue accessKeyId, HideableValue secretAccessKey) implements AwsCredential {}
+
+  record Profile(String name) implements AwsCredential {}
+
+  record Default() implements AwsCredential {}
+}

std-bits/aws/src/main/java/org/enso/aws/ClientBuilder.java

@@ -0,0 +1,48 @@
+package org.enso.aws;
+
+import org.enso.base.enso_cloud.ExternalLibrarySecretHelper;
+import org.enso.base.enso_cloud.HideableValue;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.services.s3.S3Client;
+
+public class ClientBuilder {
+  private final AwsCredential awsCredential;
+
+  public ClientBuilder(AwsCredential credential) {
+    this.awsCredential = credential;
+  }
+
+  public S3Client buildS3Client() {
+    return S3Client.builder().credentialsProvider(unsafeBuildCredentialProvider()).build();
+  }
+
+  /**
+   * The {@code AwsCredentialsProviders} may leak secrets, so it should never be returned to user
+   * code.
+   */
+  private AwsCredentialsProvider unsafeBuildCredentialProvider() {
+    return switch (this.awsCredential) {
+      case AwsCredential.Default unused -> DefaultCredentialsProvider.create();
+      case AwsCredential.Key key -> {
+        AwsBasicCredentials credentials =
+            AwsBasicCredentials.create(
+                unsafeResolveSecrets(key.accessKeyId()),
+                unsafeResolveSecrets(key.secretAccessKey()));
+        yield StaticCredentialsProvider.create(credentials);
+      }
+      case AwsCredential.Profile profile -> ProfileCredentialsProvider.create(profile.name());
+    };
+  }
+
+  /**
+   * This function is allowed access to secrets. Extra care should be taken to ensure its result is
+   * not leaked.
+   */
+  private String unsafeResolveSecrets(HideableValue value) {
+    return ExternalLibrarySecretHelper.resolveValue(value);
+  }
+}

std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java

@@ -16,28 +16,7 @@ import org.enso.base.net.URIWithSecrets;
 import org.graalvm.collections.Pair;
 
 /** Makes HTTP requests with secrets in either header or query string. */
-public class EnsoSecretHelper {
-  /**
-   * Gets the value of an HideableValue resolving secrets.
-   *
-   * @param value The value to resolve.
-   * @return The pair's value. Should not be returned to Enso.
-   */
-  private static String resolveValue(HideableValue value) {
-    return switch (value) {
-      case HideableValue.PlainValue plainValue -> plainValue.value();
-      case HideableValue.SecretValue secretValue -> {
-        yield EnsoSecretReader.readSecret(secretValue.secretId());
-      }
-      case HideableValue.ConcatValues concatValues -> {
-        String left = resolveValue(concatValues.left());
-        String right = resolveValue(concatValues.right());
-        yield left + right;
-      }
-      case HideableValue.Base64EncodeValue base64EncodeValue -> HideableValue.Base64EncodeValue
-          .encode(resolveValue(base64EncodeValue.value()));
-    };
-  }
+public final class EnsoSecretHelper extends SecretValueResolver {
 
   /** Gets a JDBC connection resolving EnsoKeyValuePair into the properties. */
   public static Connection getJDBCConnection(String url, Pair<String, HideableValue>[] properties)

std-bits/base/src/main/java/org/enso/base/enso_cloud/ExternalLibrarySecretHelper.java

@@ -0,0 +1,54 @@
+package org.enso.base.enso_cloud;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * An entry point allowing external libraries to access Enso secrets.
+ *
+ * <p>It will only allow access from trusted code locations.
+ */
+public final class ExternalLibrarySecretHelper extends SecretValueResolver {
+  public static String resolveValue(HideableValue hideableValue) throws EnsoSecretAccessDenied {
+    checkAccess();
+    return SecretValueResolver.resolveValue(hideableValue);
+  }
+
+  /**
+   * Checks the current stack trace to find the caller and checks if it is one of the allowed
+   * locations.
+   *
+   * <p>This is a very rudimentary approach to the access control, and it is not very extensible, as
+   * it requires updating std-base whenever a new library that needs access to secrets is added.
+   * However, it seems like the best simple solution for now.
+   *
+   * <p>Later we may want to replace it with some other solution, e.g. a key that trusted libraries
+   * will use to 'sign' their class name, proving that they can be trusted, without the need to
+   * update std-base whenever a new library is added.
+   */
+  private static void checkAccess() throws EnsoSecretAccessDenied {
+    var accessLocation =
+        StackWalker.getInstance()
+            .walk(
+                (stackFrameStream -> {
+                  Optional<StackWalker.StackFrame> firstClientFrame =
+                      stackFrameStream.skip(2).findFirst();
+                  if (firstClientFrame.isEmpty()) {
+                    throw new IllegalStateException("Unable to find client frame.");
+                  }
+
+                  var frame = firstClientFrame.get();
+                  return new AccessLocation(frame.getClassName(), frame.getMethodName());
+                }));
+
+    boolean isAllowed = allowedAccessLocations.contains(accessLocation);
+    if (!isAllowed) {
+      throw new EnsoSecretAccessDenied();
+    }
+  }
+
+  private record AccessLocation(String className, String method) {}
+
+  private static final List<AccessLocation> allowedAccessLocations =
+      List.of(new AccessLocation("org.enso.aws.ClientBuilder", "unsafeResolveSecrets"));
+}

std-bits/base/src/main/java/org/enso/base/enso_cloud/SecretValueResolver.java

@@ -0,0 +1,25 @@
+package org.enso.base.enso_cloud;
+
+sealed class SecretValueResolver permits EnsoSecretHelper, ExternalLibrarySecretHelper {
+  /**
+   * Gets the value of an HideableValue resolving secrets.
+   *
+   * @param value The value to resolve.
+   * @return The pair's value. Should not be returned to Enso.
+   */
+  protected static String resolveValue(HideableValue value) {
+    return switch (value) {
+      case HideableValue.PlainValue plainValue -> plainValue.value();
+      case HideableValue.SecretValue secretValue -> {
+        yield EnsoSecretReader.readSecret(secretValue.secretId());
+      }
+      case HideableValue.ConcatValues concatValues -> {
+        String left = resolveValue(concatValues.left());
+        String right = resolveValue(concatValues.right());
+        yield left + right;
+      }
+      case HideableValue.Base64EncodeValue base64EncodeValue -> HideableValue.Base64EncodeValue
+          .encode(resolveValue(base64EncodeValue.value()));
+    };
+  }
+}

test/AWS_Tests/src/S3_Spec.enso

@@ -8,6 +8,9 @@ from Standard.AWS.Errors import AWS_SDK_Error, More_Records_Available, S3_Error,
 from Standard.Test import Test, Test_Suite
 import Standard.Test.Extensions
 
+import enso_dev.Base_Tests.Network.Enso_Cloud.Cloud_Tests_Setup.Cloud_Tests_Setup
+from enso_dev.Base_Tests.Network.Enso_Cloud.Cloud_Tests_Setup import with_retries
+
 spec =
     bucket_name = "enso-data-samples"
     not_a_bucket_name = "not_a_bucket_enso"
@@ -15,6 +18,7 @@ spec =
     folder_name = "examples/"
     sub_folder_name = "examples/folder 1/"
     api_pending = if Environment.get "AWS_ACCESS_KEY_ID" . is_nothing then "No Access Key found." else Nothing
+    cloud_setup = Cloud_Tests_Setup.prepare
 
     Test.group "S3.parse_uri" <|
         Test.specify "parse bucket only uris" <|
@@ -41,6 +45,20 @@ spec =
         Test.specify "should handle auth issues" <|
             S3.list_buckets (AWS_Credential.Profile "NoSuchProfile") . should_fail_with AWS_SDK_Error
 
+        Test.specify "should not work with invalid credentials" <|
+            S3.list_buckets (AWS_Credential.Key "foo" "bar") . should_fail_with S3_Error
+
+        Test.specify "should allow to use Enso secrets within credentials" pending=cloud_setup.pending <| cloud_setup.with_prepared_environment <|
+            secret_key_id = Enso_Secret.create "my_test_secret-AWS-keyid" (Environment.get "AWS_ACCESS_KEY_ID")
+            secret_key_id.should_succeed
+            Panic.with_finalizer secret_key_id.delete <|
+                secret_key_value = Enso_Secret.create "my_test_secret-AWS-secretkey" (Environment.get "AWS_SECRET_ACCESS_KEY")
+                secret_key_value.should_succeed
+                Panic.with_finalizer secret_key_value.delete <| with_retries <|
+                    r2 = S3.list_buckets (AWS_Credential.Key secret_key_id secret_key_value)
+                    r2.should_succeed
+                    r2.should_be_a Vector
+
     ## Rest of tests need a functional S3 connection
     pending = if bucket_name.is_nothing then "No S3 bucket set." else if buckets.get.is_error then "S3 Access Failed." else if buckets.get.contains bucket_name then Nothing else "S3 Bucket Not Found."
 

test/Base_Tests/src/Network/Enso_Cloud/Secrets_Spec.enso

@@ -2,6 +2,7 @@ from Standard.Base import all
 import Standard.Base.Data.Base_64.Base_64
 import Standard.Base.Data.Enso_Cloud.Enso_Secret.Derived_Secret_Value
 import Standard.Base.Data.Enso_Cloud.Enso_Secret.Enso_Secret_Error
+import Standard.Base.Data.Enso_Cloud.Utils as Internal_Cloud_Utils
 import Standard.Base.Errors.Illegal_Argument.Illegal_Argument
 import Standard.Base.Errors.Illegal_State.Illegal_State
 import Standard.Base.Network.HTTP.Request.Request
@@ -13,6 +14,9 @@ import Standard.Test.Extensions
 import project.Network.Enso_Cloud.Cloud_Tests_Setup.Cloud_Tests_Setup
 from project.Network.Enso_Cloud.Cloud_Tests_Setup import with_retries
 
+polyglot java import org.enso.base.enso_cloud.ExternalLibrarySecretHelper
+polyglot java import org.enso.base.enso_cloud.EnsoSecretAccessDenied
+
 spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <|
     Test.group "Enso Cloud Secrets" pending=setup.pending <|
         Test.specify "should be able to list existing secrets" <|
@@ -84,7 +88,7 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <|
                     response_json.at "headers" . at "Authorization" . should_equal expected
 
         Test.specify "should allow to derive values from secrets" <|
-            secret1 = Enso_Secret.create "my_test_secret-9" "Something"
+            secret1 = Enso_Secret.create "my_test_secret-10" "Something"
             secret1.should_succeed
             Panic.with_finalizer secret1.delete <| with_retries <|
                 x = Derived_Secret_Value.from "X"
@@ -110,7 +114,7 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <|
                 b2.to_text . should_equal "base64(X__SECRET__)"
 
         Test.specify "does not allow secrets in HTTP headers" pending=setup.httpbin_pending <|
-            secret1 = Enso_Secret.create "my_test_secret-10" "Something"
+            secret1 = Enso_Secret.create "my_test_secret-11" "Something"
             secret1.should_succeed
             Panic.with_finalizer secret1.delete <| with_retries <|
                 uri = setup.httpbin_uri / "get"
@@ -118,6 +122,14 @@ spec setup:Cloud_Tests_Setup = setup.with_prepared_environment <|
                 r1.should_fail_with Illegal_Argument
                 r1.catch.to_display_text . should_contain "Secrets are not allowed in HTTP connections, use HTTPS instead."
 
+        Test.specify "API exposing secrets to external libraries should not be accessible from unauthorized code" <|
+            secret1 = Enso_Secret.create "my_test_secret-12" "Something"
+            secret1.should_succeed
+            Panic.with_finalizer secret1.delete <| with_retries <|
+                java_repr = Internal_Cloud_Utils.as_hideable_value secret1
+                Test.expect_panic EnsoSecretAccessDenied <|
+                    ExternalLibrarySecretHelper.resolveValue java_repr
+
 main = Test_Suite.run_main (spec Cloud_Tests_Setup.prepare)
 
 wait_until_secret_is_propagated secret =