Update Database.connect to match new API (#3542)

Initial work restructuring the `Database.connect` API
- New SQLite API with support for InMemory.
- Updated PostgreSQL API with SSL and Client Certificate Support.
- Updated Redshift API.

# Important Notes
Follow up tasks:
- PostgreSQL SSL additional testing.
- Driver version updating.
- `.pgpass` support.
This commit is contained in:
James Dunkerley 2022-07-04 21:26:44 +01:00 committed by GitHub
parent 1b0312b446
commit 5174cc6ece
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 349 additions and 93 deletions

View File

@ -144,6 +144,7 @@
- [Removed obsolete `from_xls` and `from_xlsx` functions. Added support for
reading column names from first row in `File_Format.Excel`][3523]
- [Added `File_Format.Delimited` support to `Table.write` for new files.][3528]
- [Adjusted `Database.connect` API to new design.][3542]
- [Added `File_Format.Excel` support to `Table.write` for new files.][3551]
[debug-shortcuts]:
@ -228,6 +229,7 @@
[3519]: https://github.com/enso-org/enso/pull/3519
[3523]: https://github.com/enso-org/enso/pull/3523
[3528]: https://github.com/enso-org/enso/pull/3528
[3542]: https://github.com/enso-org/enso/pull/3542
[3551]: https://github.com/enso-org/enso/pull/3551
#### Enso Compiler

View File

@ -0,0 +1,22 @@
from Standard.Base import all
type Client_Certificate
## Creates a new Client_Certificate object.
Arguments:
- cert_file: path to the client certificate file.
- key_file: path to the client key file.
- key_password: password for the client key file.
type Client_Certificate cert_file:(File|Text) key_file:(File|Text) (key_password:Text='')
## PRIVATE
Creates the JDBC properties for the client certificate.
JDBC Properties:
- sslcert: points to the client certificate file.
- sslkey: points to the client key file.
- sslpass: password for the client key file.
properties : Vector
properties =
base = [Pair 'sslcert' (File.new self.cert_file).absolute.path, Pair 'sslkey' (File.new self.key_file).absolute.path]
if self.key_password == "" then base else base + [Pair 'sslpassword' self.key_password]

View File

@ -56,6 +56,24 @@ type Connection
close =
self.connection_resource . finalize
## Returns the list of databases (or catalogs) for the connection.
databases : [Text]
databases =
here.wrap_sql_errors <|
self.connection_resource.with connection->
metadata = connection.getMetaData
schema_result_set = metadata.getCatalogs
Vector.Vector (JDBCProxy.getStringColumn schema_result_set "TABLE_CAT")
## Returns the list of schemas for the connection within the current database (or catalog).
schemas : [Text]
schemas =
here.wrap_sql_errors <|
self.connection_resource.with connection->
metadata = connection.getMetaData
schema_result_set = metadata.getSchemas
Vector.Vector (JDBCProxy.getStringColumn schema_result_set "TABLE_SCHEM")
## ADVANCED
Executes a raw query and returns the result as an in-memory Table.
@ -296,6 +314,7 @@ type Builder
storage = self.java_builder.seal
Java_Exports.make_column name storage
## An error indicating that a supported dialect could not be deduced for the
provided URL.
@ -316,19 +335,12 @@ Unsupported_Dialect.to_display_text =
Arguments:
- url: The URL to connect to.
- properties: A vector of properties for the connection.
create_jdbc_connection : Text -> Vector -> Connection
create_jdbc_connection url properties = here.handle_sql_errors <|
- dialect: A Dialect object that will be used to generate SQL.
create_jdbc_connection : Text -> Vector -> Dialect -> Connection
create_jdbc_connection url properties dialect = here.handle_sql_errors <|
java_props = Properties.new
properties.each pair->
java_props.setProperty pair.first pair.second
## This is a workaround for the Redshift driver - it looks for an ini file
by looking at the jar file location, which is not available in the Graal
class loader. This block may be removed when migrated to a Graal version
with https://github.com/oracle/graal/issues/3744 fixed.
if url.starts_with 'jdbc:redshift:' && (java_props.getProperty 'IniFile' . is_nothing) then
path = Enso_Project.data/'empty.ini' . absolute . path
java_props.setProperty 'IniFile' path
dialect = Dialect.supported_dialects.find (d -> url.starts_with "jdbc:"+d.name) . map_error (_ -> Unsupported_Dialect url)
java_connection = JDBCProxy.getConnection url java_props
resource = Managed_Resource.register java_connection here.close_connection
Connection resource dialect

View File

@ -0,0 +1,10 @@
from Standard.Base import all
type Connection_Options
## Hold a set of key value pairs used to configure the connection.
type Connection_Options options:Vector=[]
## Merge the base set of options with the overrides in this object.
merge : Vector -> Vector
merge base_options =
base_options.filter x->(self.options.any (y->y.first==x.first) . not) + self.options

View File

@ -0,0 +1,9 @@
from Standard.Base import all
type Credentials
## Simple username and password type.
type Credentials username:Text password:Text
## Override `to_text` to mask the password field.
to_text : Text
to_text = 'Credentials ' + self.username + ' *****'

View File

@ -1,57 +1,20 @@
from Standard.Base import all
from Standard.Database.Connection.Connection import all
from Standard.Database.Connection.Connection_Options as Connection_Options_Module import Connection_Options
import Standard.Database.Connection.PostgreSQL
import Standard.Database.Connection.SQLite
import Standard.Database.Connection.Redshift
from Standard.Database.Connection.Connection import Connection, Sql_Error
## UNSTABLE
Tries to connect to the database under a provided URL.
Tries to connect to the database.
Arguments:
- url: The URL to connect to.
- user: A username for authentication. Optional.
- password: A password for authentication. Optional.
- custom_properties: A vector of key-value Text pairs which can
set any other properties that can be used to configure the connection or
for authentication. Supported properties depend on the database engine that
the connection is made to. Optional.
Currently SQLite, PostgreSQL and Amazon Redshift are supported.
? Finding the URL
The exact URL depends on the database engine. For SQLite the expected o
format is `sqlite:/path/to/database/file`. For PostgreSQL it can be one
of:
- `postgresql:database_name` - which will connect to the database with the
given name on the local machine;
- `postgresql:/` - which will connect to the default database
(which is the same as the username) on the local machine;
- `postgresql://host/database_name` - which will connect to the specified
database on a specified host, the `host` can consist of an IP address or=
a hostname, optionally followed by colon and a port number, so values
like `db.example.com`, `127.0.0.1`, `example.com:1234`, `127.0.0.1:1234`
are allowed;
- `postgresql://host/` - which will connect to the same database as the
username on a specified host, the `host`` is defined as above.
For Redshift, the URL can be found in the cluster management section in the
AWS admin console.
connect : Text -> Nothing | Text -> Nothing | Text -> Vector -> Connection ! Sql_Error
connect url user=Nothing password=Nothing custom_properties=[] =
full_url = if url.starts_with "jdbc:" then url else "jdbc:"+url
user_prop = if user.is_nothing then [] else [["user", user]]
pass_prop = if password.is_nothing then [] else [["password", password]]
properties = user_prop + pass_prop + custom_properties
Connection.create_jdbc_connection full_url properties
## UNSTABLE
Connects to an SQLite database in a file on the filesystem.
Arguments:
- file: The path to the database.
It is an alternative to `connect` that resolves a path to the database file.
open_sqlite_file : File -> Connection ! Sql_Error
open_sqlite_file file =
url = "sqlite:" + file.absolute.path
here.connect url
- details: Connection_Details to use to connect.
- options: Any overriding options to use.
connect : (PostgreSQL|SQLite|Redshift) -> Connection_Options -> Connection ! Sql_Error
connect details options=Connection_Options =
details.connect options

View File

@ -0,0 +1,87 @@
from Standard.Base import all
import Standard.Database.Data.Dialect
import Standard.Database.Connection.Connection
from Standard.Database.Connection.Credentials as Credentials_Module import Credentials
import Standard.Database.Connection.Connection_Options
import Standard.Database.Connection.SSL_Mode
from Standard.Database.Connection.SSL_Mode import all
import Standard.Database.Connection.Client_Certificate
polyglot java import org.postgresql.Driver
type PostgreSQL
## Connect to a PostgreSQL database.
Arguments:
- host: The hostname of the database server (defaults to localhost).
- port: The port of the database server (defaults to 5432).
- database: The database to connect to. If empty, the default database will be used.
- credentials: The credentials to use for the connection (defaults to PGPass or No Authentication).
- use_ssl: Whether to use SSL (defaults to `Prefer`).
- client_cert: The client certificate to use or `Nothing` if not needed.
type PostgreSQL (host:Text='localhost') (port:Integer=5432) (database:Text='') (credentials:(Credentials|Nothing)=Nothing) (use_ssl:SSL_Mode=Prefer) (client_cert:(Client_Certificate|Nothing)=Nothing)
## Build the Connection resource.
Arguments:
- options: Overrides for the connection properties.
connect : Connection_Options
connect options =
if Driver.isRegistered.not then Driver.register
properties = options.merge self.jdbc_properties
Connection.create_jdbc_connection self.jdbc_url properties self.dialect
## Provides the jdbc url for the connection.
jdbc_url : Text
jdbc_url =
'jdbc:postgresql://' + self.host + ':' + self.port.to_text + (if self.database == '' then '' else '/' + self.database)
## Provides the properties for the connection.
jdbc_properties : [Pair Text Text]
jdbc_properties =
credentials = case self.credentials of
Nothing -> PostgreSQL.read_pgpass self.host self.port self.database
Credentials username password ->
[Pair 'user' username, Pair 'password' password]
ssl_properties = PostgreSQL.ssl_mode_to_jdbc_properties self.use_ssl
cert_properties = if self.client_cert.is_nothing then [] else
self.client_cert.properties
credentials + ssl_properties + cert_properties
## Provides the dialect needed for creating SQL statements.
dialect : Dialect
dialect = Dialect.postgres
## PRIVATE - static
Read the .pgpass file from the User's home directory and obtain username
and password. https://www.postgresql.org/docs/current/libpq-pgpass.html
Arguments:
- host: The hostname of the database server.
- port: The port of the database server.
- database: The database to connect to.
read_pgpass : Text -> Integer -> Text -> [Pair Text Text]
read_pgpass _ _ _ =
## ToDo: Code not part of the design document.
## host port database
[]
## PRIVATE - static
Given an SSL_Mode, create the JDBC properties to secure a Postgres-based
connection.
ssl_mode_to_jdbc_properties : SSL_Mode -> [Pair Text Text]
ssl_mode_to_jdbc_properties use_ssl = case use_ssl of
Disable -> []
Prefer -> [Pair 'sslmode' 'prefer']
Require -> [Pair 'sslmode' 'require']
Verify_CA cert_file ->
if cert_file.is_nothing then [Pair 'sslmode' 'verify-ca'] else
[Pair 'sslmode' 'verify-ca', Pair 'sslrootcert' (File.new cert_file).absolute.path]
Full_Verification cert_file ->
if cert_file.is_nothing then [Pair 'sslmode' 'verify-full'] else
[Pair 'sslmode' 'verify-full', Pair 'sslrootcert' (File.new cert_file).absolute.path]

View File

@ -0,0 +1,71 @@
from Standard.Base import all
import Standard.Database.Data.Dialect
import Standard.Database.Connection.Connection
from Standard.Database.Connection.Credentials as Credentials_Module import Credentials
import Standard.Database.Connection.Connection_Options
import Standard.Database.Connection.SSL_Mode
from Standard.Database.Connection.SSL_Mode import all
import Standard.Database.Connection.Client_Certificate
import Standard.Database.Connection.PostgreSQL
polyglot java import com.amazon.redshift.jdbc.Driver
type Redshift
## Connect to a AWS Redshift database.
Arguments:
- host: The hostname of the database server (defaults to localhost).
- port: The port of the database server (defaults to 5432).
- schema: The schema to connect to (if not provided or empty, the default schema will be used).
- credentials: The credentials to use for the connection (defaults to PGPass or No Authentication).
- use_ssl: Whether to use SSL (defaults to `Require`).
- client_cert: The client certificate to use or `Nothing` if not needed.
type Redshift (host:Text) (port:Integer=5439) (schema:Text='') (credentials:Credentials|AWS_Profile|Nothing=Nothing) (use_ssl:(Disable|Require|Verify_CA|Full_Verification)=Require) (client_cert:Client_Certificate|Nothing=Nothing)
## Build the Connection resource.
Arguments:
- options: Overrides for the connection properties.
connect : Connection_Options
connect options =
if Driver.isRegistered.not then Driver.register
properties = options.merge self.jdbc_properties
Connection.create_jdbc_connection self.jdbc_url properties self.dialect
## Provides the jdbc url for the connection.
jdbc_url : Text
jdbc_url =
prefix = case self.credentials of
AWS_Profile _ -> 'jdbc:redshift:iam://'
_ -> 'jdbc:redshift://'
prefix + self.host + ':' + self.port.to_text + (if self.schema == '' then '' else '/' + self.schema)
## Provides the properties for the connection.
jdbc_properties : [Pair Text Text]
jdbc_properties =
credentials = case self.credentials of
Nothing -> PostgreSQL.read_pgpass self.host self.port self.schema
AWS_Profile profile -> if profile == '' then [] else [Pair 'profile' profile]
Credentials username password ->
[Pair 'user' username, Pair 'password' password]
ssl_properties = PostgreSQL.ssl_mode_to_jdbc_properties self.use_ssl
cert_properties = if self.client_cert.is_nothing then [] else
self.client_cert.properties
## This is a workaround for the Redshift driver - it looks for an ini file
by looking at the jar file location, which is not available in the Graal
class loader. This block may be removed when migrated to a Graal version
with https://github.com/oracle/graal/issues/3744 fixed.
[Pair 'IniFile' (Enso_Project.data/'empty.ini' . absolute . path)] + credentials + ssl_properties + cert_properties
## Provides the dialect needed for creating SQL statements.
dialect : Dialect
dialect = Dialect.redshift
type AWS_Profile
type AWS_Profile profile:Text=''

View File

@ -0,0 +1,33 @@
from Standard.Base import all
import Standard.Database.Data.Dialect
import Standard.Database.Connection.Connection
import Standard.Database.Connection.Connection_Options
## Connect to a SQLite DB File or InMemory DB.
type SQLite
type SQLite (location:(InMemory|File|Text))
## Build the Connection resource.
connect : Connection_Options
connect options =
properties = options.merge self.jdbc_properties
Connection.create_jdbc_connection self.jdbc_url properties self.dialect
## Provides the jdbc url for the connection.
jdbc_url : Text
jdbc_url = case self.location of
InMemory -> "jdbc:sqlite::memory:"
_ -> "jdbc:sqlite:" + ((File.new self.location).absolute.path.replace '\\' '/')
## Provides the properties for the connection.
jdbc_properties : Vector
jdbc_properties = []
## Provides the dialect needed for creating SQL statements.
dialect : Dialect
dialect = Dialect.sqlite
## Connect to an in-memory SQLite database.
type InMemory
type InMemory

View File

@ -0,0 +1,19 @@
from Standard.Base import all
type SSL_Mode
## Do not use SSL for the connection.
type Disable
## Prefer SSL for the connection, but does not verify the server certificate.
type Prefer
## Will use SSL but does not verify the server certificate.
type Require
## Will use SSL, validating the certificate but not verifying the hostname.
If `ca_file` is `Nothing`, the default CA certificate store will be used.
type Verify_CA ca_file:Nothing|File|Text=Nothing
## Will use SSL, validating the certificate and checking the hostname matches.
If `ca_file` is `Nothing`, the default CA certificate store will be used.
type Full_Verification ca_file:Nothing|File|Text=Nothing

View File

@ -4,8 +4,8 @@ import Standard.Base.Error.Common as Errors
import Standard.Table.Data.Aggregate_Column
import Standard.Database.Data.Sql
import Standard.Database.Data.Internal.IR
import Standard.Database.Data.Dialect.Postgres
import Standard.Database.Data.Dialect.Redshift
import Standard.Database.Data.Dialect.Postgres as Postgres_Module
import Standard.Database.Data.Dialect.Redshift as Redshift_Module
import Standard.Database.Data.Dialect.Sqlite as Sqlite_Module
## PRIVATE
@ -48,14 +48,22 @@ type Dialect
prepare_order_descriptor : IR.Internal_Column -> Sort_Direction -> Text_Ordering -> IR.Order_Descriptor
prepare_order_descriptor = Errors.unimplemented "This is an interface only."
## PRIVATE
A vector of SQL dialects supported by the Database library.
supported_dialects : Vector Dialect
supported_dialects = [Postgres.postgresql, Sqlite_Module.sqlite, Redshift.redshift]
## PRIVATE
The dialect of SQLite databases.
sqlite : Dialect
sqlite = Sqlite_Module.sqlite
## PRIVATE
The dialect of PostgreSQL databases.
postgres : Dialect
postgres = Postgres_Module.postgresql
## PRIVATE
The dialect of Redshift databases.
redshift : Dialect
redshift = Redshift_Module.redshift

View File

@ -911,7 +911,7 @@ type Table
import Standard.Database
example_to_csv =
connection = Database.open_sqlite_file (File.new "db.sqlite")
connection = Database.connect (SQLite (File.new "db.sqlite"))
table = connection.access_table "Table"
table.write (Enso_Project.data / "example_csv_output.csv")
write : File|Text -> File_Format -> Existing_File_Behavior -> Column_Mapping -> Problem_Behavior -> Nothing ! Column_Mismatch | Illegal_Argument_Error | File_Not_Found | Io_Error

View File

@ -3,10 +3,25 @@ import Standard.Database.Data.Column
import Standard.Database.Connection.Connection
import Standard.Database.Connection.Database
import Standard.Database.Connection.Credentials
import Standard.Database.Connection.Client_Certificate
import Standard.Database.Connection.SSL_Mode
import Standard.Database.Connection.Connection_Options
import Standard.Database.Connection.PostgreSQL
import Standard.Database.Connection.SQLite
import Standard.Database.Connection.Redshift
export Standard.Database.Data.Table
export Standard.Database.Data.Column
export Standard.Database.Connection.Connection
from Standard.Database.Connection.Database export all
from Standard.Database.Connection.Credentials export all
from Standard.Database.Connection.Client_Certificate export all
from Standard.Database.Connection.SSL_Mode export all
from Standard.Database.Connection.Connection_Options export all
import Standard.Table.Data.Table
from Standard.Database.Connection.Database export all
from Standard.Database.Connection.PostgreSQL export all
from Standard.Database.Connection.SQLite export all
from Standard.Database.Connection.Redshift export all

View File

@ -2,7 +2,10 @@ package org.enso.database;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
/**
@ -20,8 +23,7 @@ public class JDBCProxy {
*
* @return an array of JDBC drivers that are currently registered
*/
public static Object[] getDrivers() throws SQLException {
initialize();
public static Object[] getDrivers() {
return DriverManager.drivers().toArray();
}
@ -36,16 +38,19 @@ public class JDBCProxy {
* @return a connection
*/
public static Connection getConnection(String url, Properties properties) throws SQLException {
initialize();
return DriverManager.getConnection(url, properties);
}
private static void initialize() throws SQLException {
if (!org.postgresql.Driver.isRegistered()) {
org.postgresql.Driver.register();
public static String[] getStringColumn(ResultSet resultSet, String column) throws SQLException {
if (resultSet.isClosed()) {
return new String[0];
}
if (!com.amazon.redshift.jdbc.Driver.isRegistered()) {
com.amazon.redshift.jdbc.Driver.register();
int colIndex = resultSet.findColumn(column);
List<String> values = new ArrayList<>();
while (resultSet.next()) {
values.add(resultSet.getString(colIndex));
}
return values.toArray(String[]::new);
}
}

View File

@ -114,7 +114,7 @@ run_tests connection pending=Nothing =
spec =
db_name = Environment.get "ENSO_DATABASE_TEST_DB_NAME"
db_host = Environment.get "ENSO_DATABASE_TEST_HOST"
db_host_port = (Environment.get "ENSO_DATABASE_TEST_HOST").if_nothing "localhost" . split ':'
db_user = Environment.get "ENSO_DATABASE_TEST_DB_USER"
db_password = Environment.get "ENSO_DATABASE_TEST_DB_PASSWORD"
@ -124,10 +124,8 @@ spec =
connection = Error.throw message
here.run_tests connection pending=message
False ->
url = case db_host.is_nothing of
True -> "postgresql:" + db_name
False -> "postgresql://" + db_host + "/" + db_name
connection = Database.connect url user=db_user password=db_password
db_port = if db_host_port.length == 1 then 5432 else Integer.parse (db_host_port.at 1)
connection = Database.connect (PostgreSQL (db_host_port.at 0) db_port db_name (Credentials db_user db_password))
here.run_tests connection
main = Test.Suite.run_main here.spec

View File

@ -43,13 +43,7 @@ sqlite_specific_spec connection =
t.at "reals" . sql_type . is_definitely_boolean . should_be_false
t.at "bools" . sql_type . is_definitely_double . should_be_false
spec =
Enso_Project.data.create_directory
file = Enso_Project.data / "sqlite_test.db"
file.delete_if_exists
connection = Database.open_sqlite_file file
prefix = "[SQLite] "
sqlite_spec connection prefix =
name_counter = Ref.new 0
table_builder columns =
ix = name_counter.get
@ -80,6 +74,14 @@ spec =
Aggregate_Spec.aggregate_spec prefix agg_table empty_agg_table table_builder materialize is_database=True selection
connection.close
spec =
Enso_Project.data.create_directory
file = Enso_Project.data / "sqlite_test.db"
file.delete_if_exists
here.sqlite_spec (Database.connect (SQLite file)) "[SQLite] "
file.delete
here.sqlite_spec (Database.connect (SQLite InMemory)) "[SQLite Memory] "
main = Test.Suite.run_main here.spec

View File

@ -20,7 +20,7 @@ spec =
Enso_Project.data.create_directory
file = Enso_Project.data / "sqlite_test.db"
file.delete_if_exists
connection = Database.open_sqlite_file file
connection = Database.connect (SQLite file)
here.visualization_spec connection
connection.close
file.delete

View File

@ -93,7 +93,7 @@ spec =
Enso_Project.data.create_directory
file = Enso_Project.data / "sqlite_test.db"
file.delete_if_exists
connection = Database.open_sqlite_file file
connection = Database.connect (SQLite file)
here.visualization_spec connection
connection.close
file.delete