mirror of
https://github.com/digital-asset/daml.git
synced 2024-09-17 15:57:21 +03:00
DPP-745 Add Oracle data continuity tests (#12961)
* Add Oracle compatibility tests changelog_begin changelog_end * Clean up * Remove debug change * Skip Oracle tests if there is nothing to do * Add missing line warp * Use ORACLE_PORT Co-authored-by: mziolekda <marcin.ziolek@digitalasset.com> Co-authored-by: mziolekda <marcin.ziolek@digitalasset.com>
This commit is contained in:
parent
fceb9c1118
commit
18b681f521
@ -34,3 +34,6 @@ steps:
|
||||
set -eou pipefail
|
||||
./compatibility/test.sh ${{ parameters.test_flags }}
|
||||
displayName: 'Run tests'
|
||||
env:
|
||||
DOCKER_LOGIN: $(DOCKER_LOGIN)
|
||||
DOCKER_PASSWORD: $(DOCKER_PASSWORD)
|
||||
|
33
compatibility/bazel_tools/client_server/with-oracle/BUILD
Normal file
33
compatibility/bazel_tools/client_server/with-oracle/BUILD
Normal file
@ -0,0 +1,33 @@
|
||||
load("@daml//bazel_tools:haskell.bzl", "da_haskell_binary", "da_haskell_library")
|
||||
|
||||
da_haskell_library(
|
||||
name = "with-oracle",
|
||||
srcs = ["lib/WithOracle.hs"],
|
||||
data = [],
|
||||
hackage_deps = [
|
||||
"base",
|
||||
"directory",
|
||||
"extra",
|
||||
"filepath",
|
||||
"network",
|
||||
"process",
|
||||
"safe-exceptions",
|
||||
"text",
|
||||
"uuid",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
da_haskell_binary(
|
||||
name = "with-oracle-exe",
|
||||
srcs = ["exe/Main.hs"],
|
||||
hackage_deps = [
|
||||
"base",
|
||||
"process",
|
||||
"text",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
":with-oracle",
|
||||
],
|
||||
)
|
@ -0,0 +1,16 @@
|
||||
-- Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
-- SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
module Main (main) where
|
||||
|
||||
import qualified Data.Text as T
|
||||
import System.Environment
|
||||
import System.Process
|
||||
|
||||
import WithOracle
|
||||
|
||||
main :: IO ()
|
||||
main = do
|
||||
(arg : args) <- getArgs
|
||||
withOracle $ \jdbcUrl ->
|
||||
callProcess arg (map (T.unpack . T.replace "__jdbcurl__" jdbcUrl . T.pack) args)
|
@ -0,0 +1,90 @@
|
||||
-- Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
-- SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
module WithOracle (withOracle) where
|
||||
|
||||
import Control.Exception.Safe
|
||||
import qualified Data.UUID as UUID
|
||||
import Data.UUID.V4
|
||||
import Data.Text (Text)
|
||||
import Data.Maybe
|
||||
import qualified Data.Text as T
|
||||
import Network.Socket
|
||||
import System.Environment.Blank
|
||||
import System.IO.Extra
|
||||
import System.Process
|
||||
|
||||
-- This is loosely modelled after com.daml.testing.oracle.OracleAround.
|
||||
-- We make this a separate executable since it should only
|
||||
-- depend on released artifacts so we cannot easily use sandbox as a library.
|
||||
|
||||
data OracleConnection = OracleConnection
|
||||
{ port :: PortNumber
|
||||
, adminUsername :: Text
|
||||
, adminPassword :: Text
|
||||
, dockerPath :: Text
|
||||
}
|
||||
|
||||
withOracle :: (Text -> IO a) -> IO a
|
||||
withOracle f = do
|
||||
connection <- getOracleConnection
|
||||
-- Note: we do not check whether the generated user name already exists
|
||||
username <- randomUserName
|
||||
bracket_ (createUser connection username) (dropUser connection username) $ f (jdbcUrl connection username)
|
||||
|
||||
-- Takes a UUID and modifies it so that it conforms to Oracle constraints for usernames and passwords
|
||||
randomUserName :: IO Text
|
||||
randomUserName = do fmap (T.take 30 . T.cons 'u' . T.replace "-" "" . T.pack . UUID.toString) nextRandom
|
||||
|
||||
-- Note: currently we only start Oracle databases through docker.
|
||||
-- It's easier to shell out to sqlplus than dealing with Oracle-compatible SQL libraries.
|
||||
executeStatement :: OracleConnection -> Text -> IO ()
|
||||
executeStatement OracleConnection { .. } str = do
|
||||
-- Note: need to login as 'sys', not as the user provided through the ORACLE_USERNAME environment variable
|
||||
let cmd = "echo '" <> str <> "' | " <> "sqlplus sys/" <> adminPassword <> "@ORCLPDB1 as sysdba"
|
||||
-- hPutStrLn stderr $ T.unpack $ "Executing command " <> cmd <> " through " <> dockerPath
|
||||
callProcess (T.unpack dockerPath)
|
||||
[ "exec"
|
||||
, "oracle"
|
||||
, "bash"
|
||||
, "-c"
|
||||
, T.unpack cmd
|
||||
]
|
||||
|
||||
jdbcUrl :: OracleConnection -> Text -> Text
|
||||
jdbcUrl OracleConnection { .. } username =
|
||||
"jdbc:oracle:thin:" <>
|
||||
username <>
|
||||
"/" <>
|
||||
username <>
|
||||
"@localhost:" <>
|
||||
T.pack (show port) <>
|
||||
"/ORCLPDB1"
|
||||
|
||||
getOracleConnection :: IO OracleConnection
|
||||
getOracleConnection = do
|
||||
mbPort <- getEnv "ORACLE_PORT"
|
||||
mbUser <- getEnv "ORACLE_USERNAME"
|
||||
mbPwd <- getEnv "ORACLE_PWD"
|
||||
mbDocker <- getEnv "ORACLE_DOCKER_PATH"
|
||||
let port = read $ fromMaybe (error "ORACLE_PORT environment variable not set") mbPort
|
||||
let username = T.pack $ fromMaybe (error "ORACLE_USERNAME environment variable not set") mbUser
|
||||
let password = T.pack $ fromMaybe (error "ORACLE_PWD environment variable not set") mbPwd
|
||||
let docker = T.pack $ fromMaybe (error "ORACLE_DOCKER_PATH environment variable not set") mbDocker
|
||||
pure $ OracleConnection port username password docker
|
||||
|
||||
createUser :: OracleConnection -> Text -> IO ()
|
||||
createUser connection name = do
|
||||
hPutStrLn stderr $ T.unpack $ "Creating Oracle user " <> name
|
||||
executeStatement connection $ "create user " <> name <> " identified by " <> name <> ";"
|
||||
executeStatement connection $ "grant connect, resource to " <> name <> ";"
|
||||
executeStatement connection $ "grant create table, create materialized view, create view, create procedure, create sequence, create type to " <> name <> ";"
|
||||
executeStatement connection $ "alter user " <> name <> " quota unlimited on users;"
|
||||
executeStatement connection $ "GRANT EXECUTE ON SYS.DBMS_LOCK TO " <> name <> ";"
|
||||
executeStatement connection $ "GRANT SELECT ON V_$MYSTAT TO " <> name <> ";"
|
||||
executeStatement connection $ "GRANT SELECT ON V_$LOCK TO " <> name <> ";"
|
||||
|
||||
dropUser :: OracleConnection -> Text -> IO ()
|
||||
dropUser connection name = do
|
||||
hPutStrLn stderr $ T.unpack $ "Dropping Oracle user " <> name
|
||||
executeStatement connection ("drop user " <> name <> " cascade;")
|
@ -86,6 +86,7 @@ da_haskell_binary(
|
||||
main_function = "Migration.Runner.main",
|
||||
visibility = ["//visibility:public"],
|
||||
deps = [
|
||||
"//bazel_tools/client_server/with-oracle",
|
||||
"//bazel_tools/client_server/with-postgres",
|
||||
"//bazel_tools/daml_ledger:sandbox-helper",
|
||||
"@rules_haskell//tools/runfiles",
|
||||
|
@ -19,6 +19,7 @@ import Control.Lens
|
||||
import Control.Monad
|
||||
import Data.Either
|
||||
import Data.List
|
||||
import Data.List.Extra (lower)
|
||||
import Data.Maybe
|
||||
import qualified Data.SemVer as SemVer
|
||||
import qualified Data.Text as T
|
||||
@ -37,6 +38,7 @@ import System.FilePath
|
||||
import System.IO.Extra
|
||||
import System.Process
|
||||
import WithPostgres (withPostgres)
|
||||
import WithOracle (withOracle)
|
||||
import qualified Bazel.Runfiles
|
||||
|
||||
import qualified Migration.Divulgence as Divulgence
|
||||
@ -50,15 +52,26 @@ data Options = Options
|
||||
-- ^ Ordered list of assistant binaries that will be used to run sandbox.
|
||||
-- We run through migrations in the order of the list
|
||||
, appendOnly :: AppendOnly
|
||||
, databaseType :: DatabaseType
|
||||
}
|
||||
|
||||
newtype AppendOnly = AppendOnly Bool
|
||||
data DatabaseType = Postgres | Oracle
|
||||
deriving (Eq, Ord, Show)
|
||||
|
||||
optsParser :: Parser Options
|
||||
optsParser = Options
|
||||
<$> strOption (long "model-dar")
|
||||
<*> many (strArgument mempty)
|
||||
<*> fmap AppendOnly (switch (long "append-only"))
|
||||
<*> option (eitherReader databaseTypeReader)(long "database" <> value Postgres)
|
||||
|
||||
databaseTypeReader :: String -> Either String DatabaseType
|
||||
databaseTypeReader str =
|
||||
case lower str of
|
||||
"postgres" -> Right Postgres
|
||||
"oracle" -> Right Oracle
|
||||
_ -> Left "Unknown database type. Expected postgres or oracle."
|
||||
|
||||
main :: IO ()
|
||||
main = do
|
||||
@ -69,7 +82,11 @@ main = do
|
||||
let step = Bazel.Runfiles.rlocation
|
||||
runfiles
|
||||
("compatibility" </> "sandbox-migration" </> "migration-step")
|
||||
withPostgres $ \jdbcUrl -> do
|
||||
let withDatabase = case databaseType of
|
||||
Postgres -> withPostgres
|
||||
Oracle -> withOracle
|
||||
withDatabase $ \jdbcUrl -> do
|
||||
hPutStrLn stderr $ T.unpack $ "Using database " <> jdbcUrl
|
||||
initialPlatform : _ <- pure platformAssistants
|
||||
hPutStrLn stderr "--> Uploading model DAR"
|
||||
withSandbox appendOnly initialPlatform jdbcUrl $ \p ->
|
||||
|
@ -22,6 +22,10 @@ if [[ $1 == "--append-only" ]]; then
|
||||
EXTRA_ARGS="--append-only"
|
||||
VERSIONS="${@:2}"
|
||||
fi
|
||||
if [[ $1 == "--oracle" ]]; then
|
||||
EXTRA_ARGS="--database=oracle"
|
||||
VERSIONS="${@:2}"
|
||||
fi
|
||||
SANDBOX_ARGS=""
|
||||
for PLATFORM in $VERSIONS; do
|
||||
SANDBOX_ARGS="$SANDBOX_ARGS $(rlocation daml-sdk-$PLATFORM/daml)"
|
||||
|
@ -7,6 +7,9 @@ load("//bazel_tools:versions.bzl", "versions")
|
||||
def runfiles(ver):
|
||||
return ["@daml-sdk-{}//:daml".format(ver)] + (["@daml-sdk-{}//:sandbox-on-x".format(ver)] if versions.is_at_least("2.0.0", ver) else [])
|
||||
|
||||
def oracle_versions(vers):
|
||||
return [v for v in vers if versions.is_at_least("1.18.0", v)]
|
||||
|
||||
def migration_test(name, versions, tags, quick_tags, **kwargs):
|
||||
native.sh_test(
|
||||
name = name,
|
||||
@ -36,3 +39,18 @@ def migration_test(name, versions, tags, quick_tags, **kwargs):
|
||||
args = ["--append-only"] + versions,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
# Oracle was introduced after the switch to the append-only schema
|
||||
native.sh_test(
|
||||
name = "{}-oracle".format(name),
|
||||
srcs = ["//sandbox-migration:test.sh"],
|
||||
deps = ["@bazel_tools//tools/bash/runfiles"],
|
||||
tags = tags,
|
||||
data = [
|
||||
"//sandbox-migration:sandbox-migration-runner",
|
||||
"//sandbox-migration:migration-model.dar",
|
||||
"//sandbox-migration:migration-step",
|
||||
] + [dep for ver in oracle_versions(versions) for dep in runfiles(ver)],
|
||||
args = ["--oracle"] + oracle_versions(versions),
|
||||
**kwargs
|
||||
) if len(oracle_versions(versions)) > 1 else None
|
||||
|
@ -58,6 +58,53 @@ trap stop_postgresql EXIT
|
||||
stop_postgresql # in case it's running from a previous build
|
||||
start_postgresql
|
||||
|
||||
|
||||
# Set up a shared Oracle instance
|
||||
# Note: this is code duplicated from ../ci/build.yml
|
||||
function start_oracle() {
|
||||
# Only log in automatically if DOCKER_LOGIN is set (as is in our CI).
|
||||
# When running this script locally, the developer is expected to have docker running and authenticated.
|
||||
if [ -z "${DOCKER_LOGIN:-}" ]; then
|
||||
echo "DOCKER_LOGIN is not set, skipping docker login"
|
||||
else
|
||||
docker login --username "$DOCKER_LOGIN" --password "$DOCKER_PASSWORD"
|
||||
fi
|
||||
IMAGE=$(cat ../ci/oracle_image)
|
||||
docker pull $IMAGE
|
||||
# Cleanup stray containers that might still be running from
|
||||
# another build that didn’t get shut down cleanly.
|
||||
docker rm -f oracle || true
|
||||
if [ "${OSTYPE:0:6}" = "darwin" ]; then
|
||||
# macOS: Oracle does not like if you use the host network to connect to it if it’s running in the container.
|
||||
echo "Starting oracle docker with port mapping"
|
||||
docker run -d --rm --name oracle -p $ORACLE_PORT:$ORACLE_PORT -e ORACLE_PWD=$ORACLE_PWD $IMAGE
|
||||
else
|
||||
# Unix: Oracle does not like if you connect to it via localhost if it’s running in the container.
|
||||
# Interestingly it works if you use the external IP of the host so the issue is
|
||||
# not the host it is listening on (it claims for that to be 0.0.0.0).
|
||||
# --network host is a cheap escape hatch for this.
|
||||
echo "Starting oracle docker with host network"
|
||||
docker run -d --rm --name oracle --network host -e ORACLE_PWD=$ORACLE_PWD $IMAGE
|
||||
fi
|
||||
}
|
||||
function stop_oracle() {
|
||||
docker rm -f oracle
|
||||
}
|
||||
function test_oracle_connection() {
|
||||
docker exec oracle bash -c 'sqlplus -L '"$ORACLE_USERNAME"'/'"$ORACLE_PWD"'@//localhost:'"$ORACLE_PORT"'/ORCLPDB1 <<< "select * from dba_users;"; exit $?' >/dev/null
|
||||
}
|
||||
trap stop_oracle EXIT
|
||||
start_oracle
|
||||
until test_oracle_connection
|
||||
do
|
||||
echo "Could not connect to Oracle, trying again..."
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Pass the path to the docker executable to the tests.
|
||||
# This is because the tests need to interact with the docker container started above.
|
||||
ORACLE_DOCKER_PATH=$(which docker)
|
||||
|
||||
bazel build //...
|
||||
|
||||
BAZEL_ARGS=""
|
||||
@ -69,4 +116,8 @@ bazel test //... \
|
||||
--test_env "POSTGRESQL_HOST=${POSTGRESQL_HOST}" \
|
||||
--test_env "POSTGRESQL_PORT=${POSTGRESQL_PORT}" \
|
||||
--test_env "POSTGRESQL_USERNAME=${POSTGRESQL_USERNAME}" \
|
||||
--test_env "ORACLE_PORT=${ORACLE_PORT}" \
|
||||
--test_env "ORACLE_USERNAME=${ORACLE_USERNAME}" \
|
||||
--test_env "ORACLE_PWD=${ORACLE_PWD}" \
|
||||
--test_env "ORACLE_DOCKER_PATH=${ORACLE_DOCKER_PATH}" \
|
||||
$BAZEL_ARGS
|
||||
|
Loading…
Reference in New Issue
Block a user