mirror of
https://github.com/digital-asset/daml.git
synced 2024-09-17 15:57:21 +03:00
DPP-745 Add Oracle data continuity tests (#12961)
* Add Oracle compatibility tests changelog_begin changelog_end * Clean up * Remove debug change * Skip Oracle tests if there is nothing to do * Add missing line warp * Use ORACLE_PORT Co-authored-by: mziolekda <marcin.ziolek@digitalasset.com> Co-authored-by: mziolekda <marcin.ziolek@digitalasset.com>
This commit is contained in:
parent
fceb9c1118
commit
18b681f521
@ -34,3 +34,6 @@ steps:
|
|||||||
set -eou pipefail
|
set -eou pipefail
|
||||||
./compatibility/test.sh ${{ parameters.test_flags }}
|
./compatibility/test.sh ${{ parameters.test_flags }}
|
||||||
displayName: 'Run tests'
|
displayName: 'Run tests'
|
||||||
|
env:
|
||||||
|
DOCKER_LOGIN: $(DOCKER_LOGIN)
|
||||||
|
DOCKER_PASSWORD: $(DOCKER_PASSWORD)
|
||||||
|
33
compatibility/bazel_tools/client_server/with-oracle/BUILD
Normal file
33
compatibility/bazel_tools/client_server/with-oracle/BUILD
Normal file
@ -0,0 +1,33 @@
|
|||||||
|
load("@daml//bazel_tools:haskell.bzl", "da_haskell_binary", "da_haskell_library")
|
||||||
|
|
||||||
|
da_haskell_library(
|
||||||
|
name = "with-oracle",
|
||||||
|
srcs = ["lib/WithOracle.hs"],
|
||||||
|
data = [],
|
||||||
|
hackage_deps = [
|
||||||
|
"base",
|
||||||
|
"directory",
|
||||||
|
"extra",
|
||||||
|
"filepath",
|
||||||
|
"network",
|
||||||
|
"process",
|
||||||
|
"safe-exceptions",
|
||||||
|
"text",
|
||||||
|
"uuid",
|
||||||
|
],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
)
|
||||||
|
|
||||||
|
da_haskell_binary(
|
||||||
|
name = "with-oracle-exe",
|
||||||
|
srcs = ["exe/Main.hs"],
|
||||||
|
hackage_deps = [
|
||||||
|
"base",
|
||||||
|
"process",
|
||||||
|
"text",
|
||||||
|
],
|
||||||
|
visibility = ["//visibility:public"],
|
||||||
|
deps = [
|
||||||
|
":with-oracle",
|
||||||
|
],
|
||||||
|
)
|
@ -0,0 +1,16 @@
|
|||||||
|
-- Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||||
|
-- SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
module Main (main) where
|
||||||
|
|
||||||
|
import qualified Data.Text as T
|
||||||
|
import System.Environment
|
||||||
|
import System.Process
|
||||||
|
|
||||||
|
import WithOracle
|
||||||
|
|
||||||
|
main :: IO ()
|
||||||
|
main = do
|
||||||
|
(arg : args) <- getArgs
|
||||||
|
withOracle $ \jdbcUrl ->
|
||||||
|
callProcess arg (map (T.unpack . T.replace "__jdbcurl__" jdbcUrl . T.pack) args)
|
@ -0,0 +1,90 @@
|
|||||||
|
-- Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||||
|
-- SPDX-License-Identifier: Apache-2.0
|
||||||
|
|
||||||
|
module WithOracle (withOracle) where
|
||||||
|
|
||||||
|
import Control.Exception.Safe
|
||||||
|
import qualified Data.UUID as UUID
|
||||||
|
import Data.UUID.V4
|
||||||
|
import Data.Text (Text)
|
||||||
|
import Data.Maybe
|
||||||
|
import qualified Data.Text as T
|
||||||
|
import Network.Socket
|
||||||
|
import System.Environment.Blank
|
||||||
|
import System.IO.Extra
|
||||||
|
import System.Process
|
||||||
|
|
||||||
|
-- This is loosely modelled after com.daml.testing.oracle.OracleAround.
|
||||||
|
-- We make this a separate executable since it should only
|
||||||
|
-- depend on released artifacts so we cannot easily use sandbox as a library.
|
||||||
|
|
||||||
|
data OracleConnection = OracleConnection
|
||||||
|
{ port :: PortNumber
|
||||||
|
, adminUsername :: Text
|
||||||
|
, adminPassword :: Text
|
||||||
|
, dockerPath :: Text
|
||||||
|
}
|
||||||
|
|
||||||
|
withOracle :: (Text -> IO a) -> IO a
|
||||||
|
withOracle f = do
|
||||||
|
connection <- getOracleConnection
|
||||||
|
-- Note: we do not check whether the generated user name already exists
|
||||||
|
username <- randomUserName
|
||||||
|
bracket_ (createUser connection username) (dropUser connection username) $ f (jdbcUrl connection username)
|
||||||
|
|
||||||
|
-- Takes a UUID and modifies it so that it conforms to Oracle constraints for usernames and passwords
|
||||||
|
randomUserName :: IO Text
|
||||||
|
randomUserName = do fmap (T.take 30 . T.cons 'u' . T.replace "-" "" . T.pack . UUID.toString) nextRandom
|
||||||
|
|
||||||
|
-- Note: currently we only start Oracle databases through docker.
|
||||||
|
-- It's easier to shell out to sqlplus than dealing with Oracle-compatible SQL libraries.
|
||||||
|
executeStatement :: OracleConnection -> Text -> IO ()
|
||||||
|
executeStatement OracleConnection { .. } str = do
|
||||||
|
-- Note: need to login as 'sys', not as the user provided through the ORACLE_USERNAME environment variable
|
||||||
|
let cmd = "echo '" <> str <> "' | " <> "sqlplus sys/" <> adminPassword <> "@ORCLPDB1 as sysdba"
|
||||||
|
-- hPutStrLn stderr $ T.unpack $ "Executing command " <> cmd <> " through " <> dockerPath
|
||||||
|
callProcess (T.unpack dockerPath)
|
||||||
|
[ "exec"
|
||||||
|
, "oracle"
|
||||||
|
, "bash"
|
||||||
|
, "-c"
|
||||||
|
, T.unpack cmd
|
||||||
|
]
|
||||||
|
|
||||||
|
jdbcUrl :: OracleConnection -> Text -> Text
|
||||||
|
jdbcUrl OracleConnection { .. } username =
|
||||||
|
"jdbc:oracle:thin:" <>
|
||||||
|
username <>
|
||||||
|
"/" <>
|
||||||
|
username <>
|
||||||
|
"@localhost:" <>
|
||||||
|
T.pack (show port) <>
|
||||||
|
"/ORCLPDB1"
|
||||||
|
|
||||||
|
getOracleConnection :: IO OracleConnection
|
||||||
|
getOracleConnection = do
|
||||||
|
mbPort <- getEnv "ORACLE_PORT"
|
||||||
|
mbUser <- getEnv "ORACLE_USERNAME"
|
||||||
|
mbPwd <- getEnv "ORACLE_PWD"
|
||||||
|
mbDocker <- getEnv "ORACLE_DOCKER_PATH"
|
||||||
|
let port = read $ fromMaybe (error "ORACLE_PORT environment variable not set") mbPort
|
||||||
|
let username = T.pack $ fromMaybe (error "ORACLE_USERNAME environment variable not set") mbUser
|
||||||
|
let password = T.pack $ fromMaybe (error "ORACLE_PWD environment variable not set") mbPwd
|
||||||
|
let docker = T.pack $ fromMaybe (error "ORACLE_DOCKER_PATH environment variable not set") mbDocker
|
||||||
|
pure $ OracleConnection port username password docker
|
||||||
|
|
||||||
|
createUser :: OracleConnection -> Text -> IO ()
|
||||||
|
createUser connection name = do
|
||||||
|
hPutStrLn stderr $ T.unpack $ "Creating Oracle user " <> name
|
||||||
|
executeStatement connection $ "create user " <> name <> " identified by " <> name <> ";"
|
||||||
|
executeStatement connection $ "grant connect, resource to " <> name <> ";"
|
||||||
|
executeStatement connection $ "grant create table, create materialized view, create view, create procedure, create sequence, create type to " <> name <> ";"
|
||||||
|
executeStatement connection $ "alter user " <> name <> " quota unlimited on users;"
|
||||||
|
executeStatement connection $ "GRANT EXECUTE ON SYS.DBMS_LOCK TO " <> name <> ";"
|
||||||
|
executeStatement connection $ "GRANT SELECT ON V_$MYSTAT TO " <> name <> ";"
|
||||||
|
executeStatement connection $ "GRANT SELECT ON V_$LOCK TO " <> name <> ";"
|
||||||
|
|
||||||
|
dropUser :: OracleConnection -> Text -> IO ()
|
||||||
|
dropUser connection name = do
|
||||||
|
hPutStrLn stderr $ T.unpack $ "Dropping Oracle user " <> name
|
||||||
|
executeStatement connection ("drop user " <> name <> " cascade;")
|
@ -86,6 +86,7 @@ da_haskell_binary(
|
|||||||
main_function = "Migration.Runner.main",
|
main_function = "Migration.Runner.main",
|
||||||
visibility = ["//visibility:public"],
|
visibility = ["//visibility:public"],
|
||||||
deps = [
|
deps = [
|
||||||
|
"//bazel_tools/client_server/with-oracle",
|
||||||
"//bazel_tools/client_server/with-postgres",
|
"//bazel_tools/client_server/with-postgres",
|
||||||
"//bazel_tools/daml_ledger:sandbox-helper",
|
"//bazel_tools/daml_ledger:sandbox-helper",
|
||||||
"@rules_haskell//tools/runfiles",
|
"@rules_haskell//tools/runfiles",
|
||||||
|
@ -19,6 +19,7 @@ import Control.Lens
|
|||||||
import Control.Monad
|
import Control.Monad
|
||||||
import Data.Either
|
import Data.Either
|
||||||
import Data.List
|
import Data.List
|
||||||
|
import Data.List.Extra (lower)
|
||||||
import Data.Maybe
|
import Data.Maybe
|
||||||
import qualified Data.SemVer as SemVer
|
import qualified Data.SemVer as SemVer
|
||||||
import qualified Data.Text as T
|
import qualified Data.Text as T
|
||||||
@ -37,6 +38,7 @@ import System.FilePath
|
|||||||
import System.IO.Extra
|
import System.IO.Extra
|
||||||
import System.Process
|
import System.Process
|
||||||
import WithPostgres (withPostgres)
|
import WithPostgres (withPostgres)
|
||||||
|
import WithOracle (withOracle)
|
||||||
import qualified Bazel.Runfiles
|
import qualified Bazel.Runfiles
|
||||||
|
|
||||||
import qualified Migration.Divulgence as Divulgence
|
import qualified Migration.Divulgence as Divulgence
|
||||||
@ -50,15 +52,26 @@ data Options = Options
|
|||||||
-- ^ Ordered list of assistant binaries that will be used to run sandbox.
|
-- ^ Ordered list of assistant binaries that will be used to run sandbox.
|
||||||
-- We run through migrations in the order of the list
|
-- We run through migrations in the order of the list
|
||||||
, appendOnly :: AppendOnly
|
, appendOnly :: AppendOnly
|
||||||
|
, databaseType :: DatabaseType
|
||||||
}
|
}
|
||||||
|
|
||||||
newtype AppendOnly = AppendOnly Bool
|
newtype AppendOnly = AppendOnly Bool
|
||||||
|
data DatabaseType = Postgres | Oracle
|
||||||
|
deriving (Eq, Ord, Show)
|
||||||
|
|
||||||
optsParser :: Parser Options
|
optsParser :: Parser Options
|
||||||
optsParser = Options
|
optsParser = Options
|
||||||
<$> strOption (long "model-dar")
|
<$> strOption (long "model-dar")
|
||||||
<*> many (strArgument mempty)
|
<*> many (strArgument mempty)
|
||||||
<*> fmap AppendOnly (switch (long "append-only"))
|
<*> fmap AppendOnly (switch (long "append-only"))
|
||||||
|
<*> option (eitherReader databaseTypeReader)(long "database" <> value Postgres)
|
||||||
|
|
||||||
|
databaseTypeReader :: String -> Either String DatabaseType
|
||||||
|
databaseTypeReader str =
|
||||||
|
case lower str of
|
||||||
|
"postgres" -> Right Postgres
|
||||||
|
"oracle" -> Right Oracle
|
||||||
|
_ -> Left "Unknown database type. Expected postgres or oracle."
|
||||||
|
|
||||||
main :: IO ()
|
main :: IO ()
|
||||||
main = do
|
main = do
|
||||||
@ -69,7 +82,11 @@ main = do
|
|||||||
let step = Bazel.Runfiles.rlocation
|
let step = Bazel.Runfiles.rlocation
|
||||||
runfiles
|
runfiles
|
||||||
("compatibility" </> "sandbox-migration" </> "migration-step")
|
("compatibility" </> "sandbox-migration" </> "migration-step")
|
||||||
withPostgres $ \jdbcUrl -> do
|
let withDatabase = case databaseType of
|
||||||
|
Postgres -> withPostgres
|
||||||
|
Oracle -> withOracle
|
||||||
|
withDatabase $ \jdbcUrl -> do
|
||||||
|
hPutStrLn stderr $ T.unpack $ "Using database " <> jdbcUrl
|
||||||
initialPlatform : _ <- pure platformAssistants
|
initialPlatform : _ <- pure platformAssistants
|
||||||
hPutStrLn stderr "--> Uploading model DAR"
|
hPutStrLn stderr "--> Uploading model DAR"
|
||||||
withSandbox appendOnly initialPlatform jdbcUrl $ \p ->
|
withSandbox appendOnly initialPlatform jdbcUrl $ \p ->
|
||||||
|
@ -22,6 +22,10 @@ if [[ $1 == "--append-only" ]]; then
|
|||||||
EXTRA_ARGS="--append-only"
|
EXTRA_ARGS="--append-only"
|
||||||
VERSIONS="${@:2}"
|
VERSIONS="${@:2}"
|
||||||
fi
|
fi
|
||||||
|
if [[ $1 == "--oracle" ]]; then
|
||||||
|
EXTRA_ARGS="--database=oracle"
|
||||||
|
VERSIONS="${@:2}"
|
||||||
|
fi
|
||||||
SANDBOX_ARGS=""
|
SANDBOX_ARGS=""
|
||||||
for PLATFORM in $VERSIONS; do
|
for PLATFORM in $VERSIONS; do
|
||||||
SANDBOX_ARGS="$SANDBOX_ARGS $(rlocation daml-sdk-$PLATFORM/daml)"
|
SANDBOX_ARGS="$SANDBOX_ARGS $(rlocation daml-sdk-$PLATFORM/daml)"
|
||||||
|
@ -7,6 +7,9 @@ load("//bazel_tools:versions.bzl", "versions")
|
|||||||
def runfiles(ver):
|
def runfiles(ver):
|
||||||
return ["@daml-sdk-{}//:daml".format(ver)] + (["@daml-sdk-{}//:sandbox-on-x".format(ver)] if versions.is_at_least("2.0.0", ver) else [])
|
return ["@daml-sdk-{}//:daml".format(ver)] + (["@daml-sdk-{}//:sandbox-on-x".format(ver)] if versions.is_at_least("2.0.0", ver) else [])
|
||||||
|
|
||||||
|
def oracle_versions(vers):
|
||||||
|
return [v for v in vers if versions.is_at_least("1.18.0", v)]
|
||||||
|
|
||||||
def migration_test(name, versions, tags, quick_tags, **kwargs):
|
def migration_test(name, versions, tags, quick_tags, **kwargs):
|
||||||
native.sh_test(
|
native.sh_test(
|
||||||
name = name,
|
name = name,
|
||||||
@ -36,3 +39,18 @@ def migration_test(name, versions, tags, quick_tags, **kwargs):
|
|||||||
args = ["--append-only"] + versions,
|
args = ["--append-only"] + versions,
|
||||||
**kwargs
|
**kwargs
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Oracle was introduced after the switch to the append-only schema
|
||||||
|
native.sh_test(
|
||||||
|
name = "{}-oracle".format(name),
|
||||||
|
srcs = ["//sandbox-migration:test.sh"],
|
||||||
|
deps = ["@bazel_tools//tools/bash/runfiles"],
|
||||||
|
tags = tags,
|
||||||
|
data = [
|
||||||
|
"//sandbox-migration:sandbox-migration-runner",
|
||||||
|
"//sandbox-migration:migration-model.dar",
|
||||||
|
"//sandbox-migration:migration-step",
|
||||||
|
] + [dep for ver in oracle_versions(versions) for dep in runfiles(ver)],
|
||||||
|
args = ["--oracle"] + oracle_versions(versions),
|
||||||
|
**kwargs
|
||||||
|
) if len(oracle_versions(versions)) > 1 else None
|
||||||
|
@ -58,6 +58,53 @@ trap stop_postgresql EXIT
|
|||||||
stop_postgresql # in case it's running from a previous build
|
stop_postgresql # in case it's running from a previous build
|
||||||
start_postgresql
|
start_postgresql
|
||||||
|
|
||||||
|
|
||||||
|
# Set up a shared Oracle instance
|
||||||
|
# Note: this is code duplicated from ../ci/build.yml
|
||||||
|
function start_oracle() {
|
||||||
|
# Only log in automatically if DOCKER_LOGIN is set (as is in our CI).
|
||||||
|
# When running this script locally, the developer is expected to have docker running and authenticated.
|
||||||
|
if [ -z "${DOCKER_LOGIN:-}" ]; then
|
||||||
|
echo "DOCKER_LOGIN is not set, skipping docker login"
|
||||||
|
else
|
||||||
|
docker login --username "$DOCKER_LOGIN" --password "$DOCKER_PASSWORD"
|
||||||
|
fi
|
||||||
|
IMAGE=$(cat ../ci/oracle_image)
|
||||||
|
docker pull $IMAGE
|
||||||
|
# Cleanup stray containers that might still be running from
|
||||||
|
# another build that didn’t get shut down cleanly.
|
||||||
|
docker rm -f oracle || true
|
||||||
|
if [ "${OSTYPE:0:6}" = "darwin" ]; then
|
||||||
|
# macOS: Oracle does not like if you use the host network to connect to it if it’s running in the container.
|
||||||
|
echo "Starting oracle docker with port mapping"
|
||||||
|
docker run -d --rm --name oracle -p $ORACLE_PORT:$ORACLE_PORT -e ORACLE_PWD=$ORACLE_PWD $IMAGE
|
||||||
|
else
|
||||||
|
# Unix: Oracle does not like if you connect to it via localhost if it’s running in the container.
|
||||||
|
# Interestingly it works if you use the external IP of the host so the issue is
|
||||||
|
# not the host it is listening on (it claims for that to be 0.0.0.0).
|
||||||
|
# --network host is a cheap escape hatch for this.
|
||||||
|
echo "Starting oracle docker with host network"
|
||||||
|
docker run -d --rm --name oracle --network host -e ORACLE_PWD=$ORACLE_PWD $IMAGE
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
function stop_oracle() {
|
||||||
|
docker rm -f oracle
|
||||||
|
}
|
||||||
|
function test_oracle_connection() {
|
||||||
|
docker exec oracle bash -c 'sqlplus -L '"$ORACLE_USERNAME"'/'"$ORACLE_PWD"'@//localhost:'"$ORACLE_PORT"'/ORCLPDB1 <<< "select * from dba_users;"; exit $?' >/dev/null
|
||||||
|
}
|
||||||
|
trap stop_oracle EXIT
|
||||||
|
start_oracle
|
||||||
|
until test_oracle_connection
|
||||||
|
do
|
||||||
|
echo "Could not connect to Oracle, trying again..."
|
||||||
|
sleep 1
|
||||||
|
done
|
||||||
|
|
||||||
|
# Pass the path to the docker executable to the tests.
|
||||||
|
# This is because the tests need to interact with the docker container started above.
|
||||||
|
ORACLE_DOCKER_PATH=$(which docker)
|
||||||
|
|
||||||
bazel build //...
|
bazel build //...
|
||||||
|
|
||||||
BAZEL_ARGS=""
|
BAZEL_ARGS=""
|
||||||
@ -69,4 +116,8 @@ bazel test //... \
|
|||||||
--test_env "POSTGRESQL_HOST=${POSTGRESQL_HOST}" \
|
--test_env "POSTGRESQL_HOST=${POSTGRESQL_HOST}" \
|
||||||
--test_env "POSTGRESQL_PORT=${POSTGRESQL_PORT}" \
|
--test_env "POSTGRESQL_PORT=${POSTGRESQL_PORT}" \
|
||||||
--test_env "POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME}" \
|
--test_env "POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME}" \
|
||||||
|
--test_env "ORACLE_PORT=${ORACLE_PORT}" \
|
||||||
|
--test_env "ORACLE_USERNAME=${ORACLE_USERNAME}" \
|
||||||
|
--test_env "ORACLE_PWD=${ORACLE_PWD}" \
|
||||||
|
--test_env "ORACLE_DOCKER_PATH=${ORACLE_DOCKER_PATH}" \
|
||||||
$BAZEL_ARGS
|
$BAZEL_ARGS
|
||||||
|
Loading…
Reference in New Issue
Block a user