mirror of
https://github.com/digital-asset/daml.git
synced 2024-10-05 16:57:28 +03:00
infra: local network cache (#17547)
See https://github.com/DACH-NY/daml-ci/pull/39 for details.
This commit is contained in:
Gary Verhaegen
GitHub
parent
18c9dce941
commit
0db5c8a593
@ -2,85 +2,44 @@
|
||||
# Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/../sdk"
|
||||
|
||||
## Functions
|
||||
DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
|
||||
|
||||
step() {
|
||||
echo "step: $*" >&2
|
||||
}
|
||||
cd $DIR/../sdk
|
||||
if [ -n "${BAZEL_CONFIG_DIR:-}" ]; then
|
||||
cd "$BAZEL_CONFIG_DIR"
|
||||
fi
|
||||
|
||||
is_windows() {
|
||||
[[ $os = windows ]]
|
||||
}
|
||||
# Note: date segment here has to match .bazelrc for dev machines to benefit
|
||||
# from CI cache
|
||||
CACHE_URL=http://10.0.2.10/cache/202405
|
||||
|
||||
## Main
|
||||
|
||||
# always run in the project root
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# detect the OS
|
||||
case $(uname) in
|
||||
Linux)
|
||||
os=ubuntu
|
||||
echo "build:linux --remote_cache=$CACHE_URL/ubuntu" > .bazelrc.local
|
||||
echo "build --remote_upload_local_results=true" >> .bazelrc.local
|
||||
echo "CACHE_URL=$CACHE_URL/ubuntu"
|
||||
;;
|
||||
Darwin)
|
||||
os=macos
|
||||
if [ "${IS_FORK:-}" = "False" ]; then
|
||||
GOOGLE_APPLICATION_CREDENTIALS=$(mktemp .tmp.XXXXXXXXXX)
|
||||
echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > "$GOOGLE_APPLICATION_CREDENTIALS"
|
||||
echo "build --remote_upload_local_results=true --google_credentials=$GOOGLE_APPLICATION_CREDENTIALS" > .bazelrc.local
|
||||
fi
|
||||
echo "CACHE_URL=$(cat .bazelrc | grep -o "https://.*/macos")"
|
||||
;;
|
||||
MINGW*)
|
||||
os=windows
|
||||
RULES_HASKELL_REV="$(sed -n 's/rules_haskell_version = "\(.*\)"$/\1/p' $DIR/../sdk/deps.bzl)"
|
||||
echo "Working directory: $PWD"
|
||||
echo "build --config windows" > .bazelrc.local
|
||||
SUFFIX="$(echo $PWD $RULES_HASKELL_REV | openssl dgst -md5 -r)"
|
||||
CACHE_URL=$CACHE_URL/windows/${SUFFIX:0:4}
|
||||
echo "build:windows --remote_cache=$CACHE_URL" >> .bazelrc.local
|
||||
echo "build --remote_upload_local_results=true" >> .bazelrc.local
|
||||
echo "CACHE_URL=$CACHE_URL"
|
||||
;;
|
||||
*)
|
||||
echo "unknown kernel: $(uname)"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
cd "$(dirname "$0")"/..
|
||||
|
||||
step "configuring bazel"
|
||||
|
||||
# We include the rules_haskell revision in the suffix since
|
||||
# it sometimes breaks Windows due to the lack of sandboxing.
|
||||
RULES_HASKELL_REV="$(sed -n 's/rules_haskell_version = "\(.*\)"$/\1/p' deps.bzl)"
|
||||
|
||||
if [ ! -z "${BAZEL_CONFIG_DIR:-}" ]; then
|
||||
cd "$BAZEL_CONFIG_DIR"
|
||||
fi
|
||||
|
||||
|
||||
if is_windows; then
|
||||
echo "build --config windows" > .bazelrc.local
|
||||
echo "build --config windows-ci" >> .bazelrc.local
|
||||
|
||||
# Modify the output path to avoid shared action keys.
|
||||
# The issue appears to be that GCC produces absolute paths
|
||||
# to system includes in .d files. These files are cached
|
||||
# so the absolute paths leak into different builds. Most of the time
|
||||
# this works since Azure reuses the working directory. However,
|
||||
# between the daily compatibility job seems to get a different
|
||||
# working directory because it is in a different pipeline.
|
||||
# To make matters worse, the working directory depends on which
|
||||
# job runs first afaict. There is a counter that simply gets incremented.
|
||||
# Sharing between the compatibility workspace and the main workspace
|
||||
# runs into the same issue.
|
||||
# To sidestep this we take a md5 hash of PWD
|
||||
# (this is what bazel does to determine the execroot name).
|
||||
# To avoid exceeding the maximum path limit on Windows we limit the suffix to
|
||||
# three characters.
|
||||
echo "Working directory: $PWD"
|
||||
SUFFIX="$(echo $PWD $RULES_HASKELL_REV | openssl dgst -md5 -r)"
|
||||
CACHE_URL="https://storage.googleapis.com/daml-bazel-cache/20231114/win/${SUFFIX:0:4}"
|
||||
echo "CACHE_URL=$CACHE_URL"
|
||||
echo "build:windows-ci --remote_cache=$CACHE_URL" >> .bazelrc.local
|
||||
fi
|
||||
|
||||
# sets up write access to the shared remote cache if the branch is not a fork
|
||||
if [[ "${IS_FORK}" = False ]]; then
|
||||
step "configuring write access to the remote cache"
|
||||
GOOGLE_APPLICATION_CREDENTIALS=$(mktemp .tmp.XXXXXXXXXX)
|
||||
echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > "$GOOGLE_APPLICATION_CREDENTIALS"
|
||||
unset GOOGLE_APPLICATION_CREDENTIALS_CONTENT
|
||||
export GOOGLE_APPLICATION_CREDENTIALS
|
||||
echo "build --remote_upload_local_results=true --google_credentials=${GOOGLE_APPLICATION_CREDENTIALS}" >> .bazelrc.local
|
||||
fi
|
||||
|
||||
20
sdk/.bazelrc
20
sdk/.bazelrc
@ -1,7 +1,7 @@
|
||||
# Bazel distributed cache, can be temporarily disabled by passing the following
|
||||
# flag: --noremote_accept_cached
|
||||
build:darwin --remote_cache=https://bazel-cache.da-ext.net/202309/macos
|
||||
build:linux --remote_cache=https://bazel-cache.da-ext.net/202309/ubuntu
|
||||
build:darwin --remote_cache=https://bazel-cache.da-ext.net/202405/macos
|
||||
build:linux --remote_cache=https://bazel-cache.da-ext.net/202405/ubuntu
|
||||
# Don't push local build results to the remote cache.
|
||||
build --remote_upload_local_results=false
|
||||
# Do still push local build results to the local disk cache.
|
||||
@ -85,20 +85,8 @@ build:linux --host_platform=@io_tweag_rules_nixpkgs//nixpkgs/platforms:host
|
||||
build:darwin --host_platform=@io_tweag_rules_nixpkgs//nixpkgs/platforms:host
|
||||
# and GHC's gcc on Windows
|
||||
build:windows --crosstool_top=@rules_haskell_ghc_windows_amd64//:cc_toolchain
|
||||
|
||||
# Caching is currently disabled on Windows.
|
||||
# See: https://github.com/tweag/rules_haskell/issues/744 for details.
|
||||
build:windows --noremote_accept_cached
|
||||
|
||||
# We use a special windows configuration on CI: cache is enabled. The caching
|
||||
# is not totally reliable on Windows yet re. absolute paths, but it is not a
|
||||
# problem on CI if all builds share a similar directory structure.
|
||||
build:windows-ci --remote_accept_cached=true
|
||||
|
||||
# The Windows CI machines have 4 physical and 8 logical cores. By default Bazel
|
||||
# will parallelize up to the number of logical cores. This can cause flakyness
|
||||
# in timing sensitive tests.
|
||||
build:windows-ci --jobs=4
|
||||
build:windows --remote_accept_cached=true
|
||||
build:windows --jobs=4
|
||||
|
||||
# Instruct bazel to globally use this flag for compiling protobuf files.
|
||||
# We need the source info for generating proto docs. By default proto_library strips
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
# Bazel distributed cache, can be temporarily disabled by passing the following
|
||||
# flag: --noremote_accept_cached
|
||||
build:darwin --remote_cache=https://bazel-cache.da-ext.net/202309/macos
|
||||
build:linux --remote_cache=https://bazel-cache.da-ext.net/202309/ubuntu
|
||||
build:darwin --remote_cache=https://bazel-cache.da-ext.net/202405/macos
|
||||
build:linux --remote_cache=https://bazel-cache.da-ext.net/202405/ubuntu
|
||||
# Don't push local build results to the remote cache.
|
||||
build --remote_upload_local_results=false
|
||||
# Do still push local build results to the local disk cache.
|
||||
@ -85,20 +85,8 @@ build:linux --host_platform=@io_tweag_rules_nixpkgs//nixpkgs/platforms:host
|
||||
build:darwin --host_platform=@io_tweag_rules_nixpkgs//nixpkgs/platforms:host
|
||||
# and GHC's gcc on Windows
|
||||
build:windows --crosstool_top=@rules_haskell_ghc_windows_amd64//:cc_toolchain
|
||||
|
||||
# Caching is currently disabled on Windows.
|
||||
# See: https://github.com/tweag/rules_haskell/issues/744 for details.
|
||||
build:windows --noremote_accept_cached
|
||||
|
||||
# We use a special windows configuration on CI: cache is enabled. The caching
|
||||
# is not totally reliable on Windows yet re. absolute paths, but it is not a
|
||||
# problem on CI if all builds share a similar directory structure.
|
||||
build:windows-ci --remote_accept_cached=true
|
||||
|
||||
# The Windows CI machines have 4 physical and 8 logical cores. By default Bazel
|
||||
# will parallelize up to the number of logical cores. This can cause flakyness
|
||||
# in timing sensitive tests.
|
||||
build:windows-ci --jobs=4
|
||||
build:windows --remote_accept_cached=true
|
||||
build:windows --jobs=4
|
||||
|
||||
# Instruct bazel to globally use this flag for compiling protobuf files.
|
||||
# We need the source info for generating proto docs. By default proto_library strips
|
||||
|
||||
Loading…
Reference in New Issue
Block a user