Update SQLite, PostgreSQL and Redshift drivers (#3571)

Updated the SQLite, PostgreSQL and Redshift drivers.

# Important Notes
Updated the API for Redshift and proved able to connect without the ini file workaround.

build.sbt

@@ -1827,11 +1827,12 @@ lazy val `std-database` = project
     Compile / packageBin / artifactPath :=
       `database-polyglot-root` / "std-database.jar",
     libraryDependencies ++= Seq(
-      "org.xerial"          % "sqlite-jdbc"           % "3.34.0",
+      "org.xerial"          % "sqlite-jdbc"           % "3.36.0.3",
-      "org.postgresql"      % "postgresql"            % "42.2.19",
+      "org.postgresql"      % "postgresql"            % "42.3.6",
-      "com.amazon.redshift" % "redshift-jdbc42"       % "2.0.0.7",
+      "com.amazon.redshift" % "redshift-jdbc42"       % "2.1.0.1",
-      "com.amazonaws"       % "aws-java-sdk-core"     % "1.12.58",
+      "com.amazonaws"       % "aws-java-sdk-core"     % "1.12.23",
-      "com.amazonaws"       % "aws-java-sdk-redshift" % "1.12.58"
+      "com.amazonaws"       % "aws-java-sdk-redshift" % "1.12.23",
+      "com.amazonaws"       % "aws-java-sdk-sts"      % "1.12.23"
     ),
     Compile / packageBin := Def.task {
       val result = (Compile / packageBin).value
@@ -1839,8 +1840,7 @@ lazy val `std-database` = project
         .copyDependencies(
           `database-polyglot-root`,
           Some("std-database.jar"),
-          ignoreScalaLibrary = true,
+          ignoreScalaLibrary = true
-          unpackedDeps       = Set("aws-java-sdk-core", "httpclient")
         )
         .value
       result

distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/NOTICE

@@ -3,22 +3,27 @@ Copyright 2020 - 2022 New Byte Order sp. z o. o.
 
 'redshift-jdbc42', licensed under the Apache License, Version 2.0, is distributed with the Database.
 The license file can be found at `licenses/APACHE2.0`.
-Copyright notices related to this dependency can be found in the directory `com.amazon.redshift.redshift-jdbc42-2.0.0.7`.
+Copyright notices related to this dependency can be found in the directory `com.amazon.redshift.redshift-jdbc42-2.1.0.1`.
 
 
 'aws-java-sdk-core', licensed under the Apache License, Version 2.0, is distributed with the Database.
 The license file can be found at `licenses/APACHE2.0`.
-Copyright notices related to this dependency can be found in the directory `com.amazonaws.aws-java-sdk-core-1.12.58`.
+Copyright notices related to this dependency can be found in the directory `com.amazonaws.aws-java-sdk-core-1.12.23`.
 
 
 'aws-java-sdk-redshift', licensed under the Apache License, Version 2.0, is distributed with the Database.
 The license file can be found at `licenses/APACHE2.0`.
-Copyright notices related to this dependency can be found in the directory `com.amazonaws.aws-java-sdk-redshift-1.12.58`.
+Copyright notices related to this dependency can be found in the directory `com.amazonaws.aws-java-sdk-redshift-1.12.23`.
+
+
+'aws-java-sdk-sts', licensed under the Apache License, Version 2.0, is distributed with the Database.
+The license file can be found at `licenses/APACHE2.0`.
+Copyright notices related to this dependency can be found in the directory `com.amazonaws.aws-java-sdk-sts-1.12.23`.
 
 
 'jmespath-java', licensed under the Apache License, Version 2.0, is distributed with the Database.
 The license file can be found at `licenses/APACHE2.0`.
-Copyright notices related to this dependency can be found in the directory `com.amazonaws.jmespath-java-1.12.58`.
+Copyright notices related to this dependency can be found in the directory `com.amazonaws.jmespath-java-1.12.23`.
 
 
 'jackson-annotations', licensed under the The Apache Software License, Version 2.0, is distributed with the Database.
@@ -73,12 +78,12 @@ Copyright notices related to this dependency can be found in the directory `org.
 
 'postgresql', licensed under the BSD-2-Clause, is distributed with the Database.
 The license information can be found along with the copyright notices.
-Copyright notices related to this dependency can be found in the directory `org.postgresql.postgresql-42.2.19`.
+Copyright notices related to this dependency can be found in the directory `org.postgresql.postgresql-42.3.6`.
 
 
 'sqlite-jdbc', licensed under the The Apache Software License, Version 2.0, is distributed with the Database.
 The license information can be found along with the copyright notices.
-Copyright notices related to this dependency can be found in the directory `org.xerial.sqlite-jdbc-3.34.0`.
+Copyright notices related to this dependency can be found in the directory `org.xerial.sqlite-jdbc-3.36.0.3`.
 
 
 'ion-java', licensed under the The Apache License, Version 2.0, is distributed with the Database.

distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/com.amazonaws.aws-java-sdk-sts-1.12.23/NOTICES

@@ -0,0 +1,13 @@
+Copyright 2010-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+Copyright 2011-2021 Amazon Technologies, Inc.
+
+Copyright 2011-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+Copyright 2012-2021 Amazon Technologies, Inc.
+
+Copyright 2013-2021 Amazon Technologies, Inc.
+
+Copyright 2014-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+Copyright 2016-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.

distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/com.amazonaws.jmespath-java-1.12.23/NOTICES

@@ -0,0 +1,14 @@
+/*
+ * Copyright 2010-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */

distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.postgresql.postgresql-42.3.6/LICENSE

@@ -0,0 +1,23 @@
+Copyright (c) 1997, PostgreSQL Global Development Group
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+   this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.

distribution/lib/Standard/Database/0.0.0-dev/THIRD-PARTY/org.xerial.sqlite-jdbc-3.36.0.3/NOTICES

@@ -31,6 +31,22 @@
  */
 
 
+/*
+ * Copyright (c) 2021 Gauthier Roebroeck <gauthier.roebroeck@gmail.com>
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
 Copyright (c) 2007 David Crawshaw <david@zentus.com>
 
 Copyright 2007 Taro L. Saito

distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Redshift.enso

@@ -11,6 +11,8 @@ import Standard.Database.Connection.Client_Certificate
 import Standard.Database.Connection.PostgreSQL
 
 polyglot java import com.amazon.redshift.jdbc.Driver
+polyglot java import java.util.Properties
+polyglot java import org.enso.database.JDBCProxy
 
 type Redshift
     ## Connect to a AWS Redshift database.
@@ -33,13 +35,17 @@ type Redshift
         if Driver.isRegistered.not then Driver.register
 
         properties = options.merge self.jdbc_properties
+        java_props = Properties.new
+        properties.each pair->
+            java_props.setProperty pair.first pair.second
         Connection.create_jdbc_connection self.jdbc_url properties self.dialect
 
     ## Provides the jdbc url for the connection.
     jdbc_url : Text
     jdbc_url =
         prefix = case self.credentials of
-            AWS_Profile _ -> 'jdbc:redshift:iam://'
+            AWS_Profile _ _ -> 'jdbc:redshift:iam://'
+            AWS_Key _ _ _ -> 'jdbc:redshift:iam://'
             _ -> 'jdbc:redshift://'
         prefix + self.host + ':' + self.port.to_text + (if self.schema == '' then '' else '/' + self.schema)
 
@@ -47,25 +53,40 @@ type Redshift
     jdbc_properties : [Pair Text Text]
     jdbc_properties =
         credentials = case self.credentials of
-            Nothing -> PostgreSQL.read_pgpass self.host self.port self.schema
+            Nothing -> PostgreSQL.PostgreSQL.read_pgpass self.host self.port self.schema
-            AWS_Profile profile -> if profile == '' then [] else [Pair 'profile' profile]
+            AWS_Profile db_user profile ->
+                [Pair 'user' db_user] + (if profile == '' then [] else [Pair 'profile' profile])
+            AWS_Key db_user access_key secret_access_key ->
+                [Pair 'user' db_user, Pair 'AccessKeyID' access_key, Pair 'SecretAccessKey' secret_access_key]
             Credentials username password ->
                 [Pair 'user' username, Pair 'password' password]
 
-        ssl_properties = PostgreSQL.ssl_mode_to_jdbc_properties self.use_ssl
+        ## Disabled as Redshift SSL settings are different to PostgreSQL.
+        ## ssl_properties = PostgreSQL.PostgreSQL.ssl_mode_to_jdbc_properties self.use_ssl
+        ssl_properties = []
 
         cert_properties = if self.client_cert.is_nothing then [] else
             self.client_cert.properties
 
-        ## This is a workaround for the Redshift driver - it looks for an ini file
+        credentials + ssl_properties + cert_properties
-           by looking at the jar file location, which is not available in the Graal
-           class loader. This block may be removed when migrated to a Graal version
-           with https://github.com/oracle/graal/issues/3744 fixed.
-        [Pair 'IniFile' (enso_project.data/'empty.ini' . absolute . path)] + credentials + ssl_properties + cert_properties
 
     ## Provides the dialect needed for creating SQL statements.
     dialect : Dialect
     dialect = Dialect.redshift
 
 type AWS_Profile
-    type AWS_Profile profile:Text=''
+    ## Access Redshift using IAM via an AWS profile.
+    
+       Arguments:
+       - db_user: Redshift username to connect as.
+       - profile: AWS profile name (if empty uses default).
+    type AWS_Profile db_user:Text profile:Text=''
+
+
+    ## Access Redshift using IAM via an AWS access key ID and secret access key.
+    
+       Arguments:
+       - db_user: Redshift username to connect as.
+       - access_key: AWS access key ID.
+       - secret_access_key: AWS secret access key.
+    type AWS_Key db_user:Text access_key:Text secret_access_key:Text

test/Table_Tests/src/Database/Redshift_Spec.enso

@@ -66,21 +66,59 @@ run_tests connection pending=Nothing =
 
     clean_tables tables.to_vector
 
-spec =
+connect_via_json_config =
     credentials = enso_project.data / 'redshift_credentials.json'
-    case credentials.exists of
+    msg = "Redshift connection is not set up. Please create a JSON file containing the credentials in `data/redshift_credentials.json`"
-        True ->
+
-            creds = Json.parse credentials.read_text . unwrap
+    if credentials.exists.not then msg else
-            access_key = ['AccessKeyID', creds.get 'access_key_id']
+        creds = Json.parse credentials.read_text . unwrap
-            secret_key = ['SecretAccessKey', creds.get 'secret_access_key']
+        access_key = creds.get 'access_key_id'
-            uri = creds.get 'db_uri'
+        secret_key = creds.get 'secret_access_key'
-            user = creds.get 'db_user'
+        uri = uri_parse (creds.get 'db_uri')
-            props = [access_key, secret_key]
+        db_uri = uri.at 0
-            connection = Database.connect uri user=user custom_properties=props
+        db_port = uri.at 1
+        db_name = uri.at 2
+
+        user = creds.get 'db_user'
+        Redshift db_uri db_port db_name credentials=(AWS_Key user access_key secret_key)
+
+connect_via_aws_environment db_host_port =
+    db_host_port_split = uri_parse db_host_port
+    db_uri = db_host_port_split.at 0
+    db_port = db_host_port_split.at 1
+    db_name = db_host_port_split.at 2
+
+    db_user = Environment.get "ENSO_REDSHIFT_USER"
+    access_key = Environment.get "AWS_ACCESS_KEY_ID"
+    secret_key = Environment.get "AWS_SECRET_ACCESS_KEY"
+
+    credentials = if (access_key.is_nothing || secret_key.is_nothing) then AWS_Profile db_user (Environment.get "AWS_PROFILE" . if_nothing '') else
+        AWS_Key db_user access_key secret_key
+
+    Redshift db_uri db_port db_name credentials=credentials
+
+uri_parse uri =
+    host_db_split = uri.split '/'
+    host_split = host_db_split.at 0 . split ':'
+
+    db_host = host_split.first
+    db_port = if host_split.length == 1 then 5439 else
+        Integer.parse (host_split.at 1)
+
+    db_name = if host_db_split.length == 1 then '' else host_db_split.at 1
+    [db_host, db_port, db_name]
+
+spec =
+    db_host_port = Environment.get "ENSO_REDSHIFT_URI"
+    connection_details = if db_host_port.is_nothing then connect_via_json_config else
+        connect_via_aws_environment db_host_port
+
+    case connection_details of
+        Text ->
+            connection = Error.throw connection_details
+            run_tests connection pending=connection_details
+        _ ->
+            connection = Database.connect connection_details
             run_tests connection
-        False ->
+
-            msg = """
+main = Test.Suite.run_main spec
-                Redshift connection is not set up. Please create a JSON file containing
-                the credentials in `data/redshift_credentials.json`
-            connection = Error.throw msg
-            run_tests connection pending=msg

tools/legal-review/Database/com.amazonaws.aws-java-sdk-sts-1.12.23/copyright-keep

@@ -0,0 +1,7 @@
+Copyright 2010-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2011-2021 Amazon Technologies, Inc.
+Copyright 2011-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2012-2021 Amazon Technologies, Inc.
+Copyright 2013-2021 Amazon Technologies, Inc.
+Copyright 2014-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Copyright 2016-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.

tools/legal-review/Database/com.amazonaws.jmespath-java-1.12.58/copyright-keep

@@ -1 +0,0 @@
-Copyright 2010-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.

tools/legal-review/Database/org.postgresql.postgresql-42.3.6/files-keep

@@ -0,0 +1 @@
+META-INF/LICENSE

tools/legal-review/Database/org.xerial.sqlite-jdbc-3.36.0.3/copyright-keep-context

@@ -0,0 +1 @@
+Copyright (c) 2021 Gauthier Roebroeck <gauthier.roebroeck@gmail.com>

tools/legal-review/Database/report-state

@@ -1,3 +1,3 @@
-E4CED7F4157F4F74DE4F190880214B959C9B605D908F22F086186085AAA5BE77
+09E39742F3F5DB02254F00E0BE2FFA52D28E8432EF731A37E58F094E33B11E4E
-EF8D764C77E3597A95CEFA0E8D39EBC73733C0B1029820E2470ABA1DCC1560B9
+29AEA17C6A4FD419982B17CF7361F23909FB0812810EB61EA66AB314605249BF
 0