mirror of
https://github.com/digital-asset/daml.git
synced 2024-09-17 07:47:14 +03:00
ci/cron/check: low-hanging perf improvement (#9042)
Two quick improvements I made while waiting on #9039: - Avoid loading Java. When looking at the logs flow by this seemed to be taking a huge amount of time. - Isolate the gcloud config files, which allows for running gcloud downloads in parallel. Together these reduce the `check_releases` runtime from about 5 hours to about 2. There's much more (and smarter) work needed on this, but this was really easy to do. CHANGELOG_BEGIN CHANGELOG_END
This commit is contained in:
parent
fb0bd82740
commit
121534c54d
@ -81,9 +81,11 @@ steps:
|
||||
key=$(mktemp)
|
||||
# There may already be a trap; this will save it
|
||||
restore_trap=$(trap -p EXIT)
|
||||
cleanup="rm -rf $key ~/.config/gcloud"
|
||||
trap "$cleanup" EXIT
|
||||
config_dir=$(mktemp -d)
|
||||
cleanup="rm -rf $key $config_dir"
|
||||
trap "$cleanup; $restore_trap" EXIT
|
||||
echo "$cred" > $key
|
||||
export CLOUDSDK_CONFIG="$config_dir"
|
||||
gcloud auth activate-service-account --key-file=$key
|
||||
|
||||
BOTO_CONFIG=/dev/null gsutil $cmd "${args[@]}"
|
||||
|
@ -344,6 +344,7 @@ verify_signatures :: FilePath -> FilePath -> String -> IO ()
|
||||
verify_signatures bash_lib tmp version_tag = do
|
||||
System.callCommand $ unlines ["bash -c '",
|
||||
"set -euo pipefail",
|
||||
"export DADE_SKIP_JAVA=1",
|
||||
"eval \"$(dev-env/bin/dade assist)\"",
|
||||
"source \"" <> bash_lib <> "\"",
|
||||
"shopt -s extglob", -- enable !() pattern: things that _don't_ match
|
||||
@ -370,6 +371,7 @@ does_backup_exist :: String -> FilePath -> FilePath -> IO Bool
|
||||
does_backup_exist gcp_credentials bash_lib path = do
|
||||
out <- shell $ unlines ["bash -c '",
|
||||
"set -euo pipefail",
|
||||
"export DADE_SKIP_JAVA=1",
|
||||
"eval \"$(dev-env/bin/dade assist)\"",
|
||||
"source \"" <> bash_lib <> "\"",
|
||||
"GCRED=$(cat <<END",
|
||||
@ -388,6 +390,7 @@ gcs_cp :: String -> FilePath -> FilePath -> FilePath -> IO ()
|
||||
gcs_cp gcp_credentials bash_lib local_path remote_path = do
|
||||
shell_ $ unlines ["bash -c '",
|
||||
"set -euo pipefail",
|
||||
"export DADE_SKIP_JAVA=1",
|
||||
"eval \"$(dev-env/bin/dade assist)\"",
|
||||
"source \"" <> bash_lib <> "\"",
|
||||
"GCRED=$(cat <<END",
|
||||
@ -417,8 +420,9 @@ check_releases gcp_credentials bash_lib max_releases = do
|
||||
IO.withTempDir $ \temp_dir -> do
|
||||
download_assets temp_dir release
|
||||
verify_signatures bash_lib temp_dir v
|
||||
Control.Monad.Extra.whenJust gcp_credentials $ \gcred ->
|
||||
Directory.listDirectory temp_dir >>= Data.Foldable.traverse_ (\f -> do
|
||||
Control.Monad.Extra.whenJust gcp_credentials $ \gcred -> do
|
||||
files <- Directory.listDirectory temp_dir
|
||||
Control.Concurrent.Async.forConcurrently_ files $ \f -> do
|
||||
let local_github = temp_dir </> f
|
||||
let local_gcp = temp_dir </> f <> ".gcp"
|
||||
let remote_gcp = "gs://daml-data/releases/" <> v <> "/github/" <> f
|
||||
@ -429,7 +433,7 @@ check_releases gcp_credentials bash_lib max_releases = do
|
||||
True -> putStrLn $ f <> " matches GCS backup."
|
||||
False -> Exit.die $ f <> " does not match GCS backup."
|
||||
else do
|
||||
Exit.die $ remote_gcp <> " does not exist. Aborting."))
|
||||
Exit.die $ remote_gcp <> " does not exist. Aborting.")
|
||||
|
||||
data CliArgs = Docs
|
||||
| Check { bash_lib :: String,
|
||||
|
@ -2,7 +2,7 @@
|
||||
|
||||
DADE_CURRENT_SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
source "$DADE_CURRENT_SCRIPT_DIR/../lib/dade-common"
|
||||
|
||||
linkTool java out "${DADE_DEVENV_DIR}/jdk"
|
||||
|
||||
if [ -z "${DADE_SKIP_JAVA+1}" ]; then
|
||||
linkTool java out "${DADE_DEVENV_DIR}/jdk"
|
||||
fi
|
||||
exec "${DADE_DEVENV_DIR}/lib/dade-dump-profile"
|
||||
|
Loading…
Reference in New Issue
Block a user