mirror of
https://github.com/digital-asset/daml.git
synced 2024-09-17 15:57:21 +03:00
ci/cron/check: low-hanging perf improvement (#9042)
Two quick improvements I made while waiting on #9039: - Avoid loading Java. When looking at the logs flow by this seemed to be taking a huge amount of time. - Isolate the gcloud config files, which allows for running gcloud downloads in parallel. Together these reduce the `check_releases` runtime from about 5 hours to about 2. There's much more (and smarter) work needed on this, but this was really easy to do. CHANGELOG_BEGIN CHANGELOG_END
This commit is contained in:
parent
fb0bd82740
commit
121534c54d
@ -81,9 +81,11 @@ steps:
|
|||||||
key=$(mktemp)
|
key=$(mktemp)
|
||||||
# There may already be a trap; this will save it
|
# There may already be a trap; this will save it
|
||||||
restore_trap=$(trap -p EXIT)
|
restore_trap=$(trap -p EXIT)
|
||||||
cleanup="rm -rf $key ~/.config/gcloud"
|
config_dir=$(mktemp -d)
|
||||||
trap "$cleanup" EXIT
|
cleanup="rm -rf $key $config_dir"
|
||||||
|
trap "$cleanup; $restore_trap" EXIT
|
||||||
echo "$cred" > $key
|
echo "$cred" > $key
|
||||||
|
export CLOUDSDK_CONFIG="$config_dir"
|
||||||
gcloud auth activate-service-account --key-file=$key
|
gcloud auth activate-service-account --key-file=$key
|
||||||
|
|
||||||
BOTO_CONFIG=/dev/null gsutil $cmd "${args[@]}"
|
BOTO_CONFIG=/dev/null gsutil $cmd "${args[@]}"
|
||||||
|
@ -344,6 +344,7 @@ verify_signatures :: FilePath -> FilePath -> String -> IO ()
|
|||||||
verify_signatures bash_lib tmp version_tag = do
|
verify_signatures bash_lib tmp version_tag = do
|
||||||
System.callCommand $ unlines ["bash -c '",
|
System.callCommand $ unlines ["bash -c '",
|
||||||
"set -euo pipefail",
|
"set -euo pipefail",
|
||||||
|
"export DADE_SKIP_JAVA=1",
|
||||||
"eval \"$(dev-env/bin/dade assist)\"",
|
"eval \"$(dev-env/bin/dade assist)\"",
|
||||||
"source \"" <> bash_lib <> "\"",
|
"source \"" <> bash_lib <> "\"",
|
||||||
"shopt -s extglob", -- enable !() pattern: things that _don't_ match
|
"shopt -s extglob", -- enable !() pattern: things that _don't_ match
|
||||||
@ -370,6 +371,7 @@ does_backup_exist :: String -> FilePath -> FilePath -> IO Bool
|
|||||||
does_backup_exist gcp_credentials bash_lib path = do
|
does_backup_exist gcp_credentials bash_lib path = do
|
||||||
out <- shell $ unlines ["bash -c '",
|
out <- shell $ unlines ["bash -c '",
|
||||||
"set -euo pipefail",
|
"set -euo pipefail",
|
||||||
|
"export DADE_SKIP_JAVA=1",
|
||||||
"eval \"$(dev-env/bin/dade assist)\"",
|
"eval \"$(dev-env/bin/dade assist)\"",
|
||||||
"source \"" <> bash_lib <> "\"",
|
"source \"" <> bash_lib <> "\"",
|
||||||
"GCRED=$(cat <<END",
|
"GCRED=$(cat <<END",
|
||||||
@ -388,6 +390,7 @@ gcs_cp :: String -> FilePath -> FilePath -> FilePath -> IO ()
|
|||||||
gcs_cp gcp_credentials bash_lib local_path remote_path = do
|
gcs_cp gcp_credentials bash_lib local_path remote_path = do
|
||||||
shell_ $ unlines ["bash -c '",
|
shell_ $ unlines ["bash -c '",
|
||||||
"set -euo pipefail",
|
"set -euo pipefail",
|
||||||
|
"export DADE_SKIP_JAVA=1",
|
||||||
"eval \"$(dev-env/bin/dade assist)\"",
|
"eval \"$(dev-env/bin/dade assist)\"",
|
||||||
"source \"" <> bash_lib <> "\"",
|
"source \"" <> bash_lib <> "\"",
|
||||||
"GCRED=$(cat <<END",
|
"GCRED=$(cat <<END",
|
||||||
@ -417,8 +420,9 @@ check_releases gcp_credentials bash_lib max_releases = do
|
|||||||
IO.withTempDir $ \temp_dir -> do
|
IO.withTempDir $ \temp_dir -> do
|
||||||
download_assets temp_dir release
|
download_assets temp_dir release
|
||||||
verify_signatures bash_lib temp_dir v
|
verify_signatures bash_lib temp_dir v
|
||||||
Control.Monad.Extra.whenJust gcp_credentials $ \gcred ->
|
Control.Monad.Extra.whenJust gcp_credentials $ \gcred -> do
|
||||||
Directory.listDirectory temp_dir >>= Data.Foldable.traverse_ (\f -> do
|
files <- Directory.listDirectory temp_dir
|
||||||
|
Control.Concurrent.Async.forConcurrently_ files $ \f -> do
|
||||||
let local_github = temp_dir </> f
|
let local_github = temp_dir </> f
|
||||||
let local_gcp = temp_dir </> f <> ".gcp"
|
let local_gcp = temp_dir </> f <> ".gcp"
|
||||||
let remote_gcp = "gs://daml-data/releases/" <> v <> "/github/" <> f
|
let remote_gcp = "gs://daml-data/releases/" <> v <> "/github/" <> f
|
||||||
@ -429,7 +433,7 @@ check_releases gcp_credentials bash_lib max_releases = do
|
|||||||
True -> putStrLn $ f <> " matches GCS backup."
|
True -> putStrLn $ f <> " matches GCS backup."
|
||||||
False -> Exit.die $ f <> " does not match GCS backup."
|
False -> Exit.die $ f <> " does not match GCS backup."
|
||||||
else do
|
else do
|
||||||
Exit.die $ remote_gcp <> " does not exist. Aborting."))
|
Exit.die $ remote_gcp <> " does not exist. Aborting.")
|
||||||
|
|
||||||
data CliArgs = Docs
|
data CliArgs = Docs
|
||||||
| Check { bash_lib :: String,
|
| Check { bash_lib :: String,
|
||||||
|
@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
DADE_CURRENT_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
DADE_CURRENT_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||||
source "$DADE_CURRENT_SCRIPT_DIR/../lib/dade-common"
|
source "$DADE_CURRENT_SCRIPT_DIR/../lib/dade-common"
|
||||||
|
if [ -z "${DADE_SKIP_JAVA+1}" ]; then
|
||||||
linkTool java out "${DADE_DEVENV_DIR}/jdk"
|
linkTool java out "${DADE_DEVENV_DIR}/jdk"
|
||||||
|
fi
|
||||||
exec "${DADE_DEVENV_DIR}/lib/dade-dump-profile"
|
exec "${DADE_DEVENV_DIR}/lib/dade-dump-profile"
|
||||||
|
Loading…
Reference in New Issue
Block a user