mirror of
https://github.com/digital-asset/daml.git
synced 2024-10-05 16:57:28 +03:00
rewrite check-releases in Bash (#16260)
This commit is contained in:
Gary Verhaegen
GitHub
parent
5e35ff283b
commit
8435730ac3
105
ci/bash-lib.yml
105
ci/bash-lib.yml
@ -115,111 +115,6 @@ steps:
|
||||
|
||||
wrap_gcloud "$cred" "gsutil $cmd"
|
||||
)
|
||||
gpg_verify() {
|
||||
local key gpg_dir signature_file res
|
||||
signature_file=$1
|
||||
key=$(mktemp)
|
||||
cat > $key <<PUB_KEY
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQINBGO9khIBEAC/D5WTgMJQGQso1JfN5RTq6YiCBwJ+L84YfKCPUo1yW7/RQHNZ
|
||||
+5rYUQpGf1K5KCIhHtJeQyANzPy9KWnhDX6lIaoau6Dg9JK3SwNv20jDyCzZOjNW
|
||||
Gfajy7xVTWXmYM/us8/A5kJN4pwEGIUL73n2uOtOzhpJ6TGLujNKB5EfGUO1L2Jr
|
||||
v9BGx2ghv+dbdR3kPX6SYuj7U+tDvoaqJB8729kL14grpBqYy2YhF5eoLyvBaE9x
|
||||
brDydUCu5t2Xpr7yI7xGOhUSn2ygoP3e9YSjOhowj5U5oFtTGxvqSf7xd9gkFaZY
|
||||
uA58X3su0nxZ/9nbvb2RJPKtlUeOJS8pggXVSSGrHfWw3Bnu2G1pQNO+MYCS0Cu/
|
||||
gMxQTnJ4itUNoFb3c9dSnB/VXWxsvlK3F+EdFg9HLNiStJVxPhPwgTo138ohTI1H
|
||||
4eGdXpRPZSKNXGRRtWdbEseYBSDBzR0ulAn5TDXFDFjjJ5u7KJfdN7p9YaXWkXpB
|
||||
+hvsiWJuvUDxTGlQE02PQjyN5vzj1NaU7CRRLvOYSstsOyTmuYg/xxvqA9XbPdti
|
||||
g9AtaeYSjRzq7OBq79FhcmKDOfh7Zc07RRXHy2xTdvw+Iy5HEjk0fYFz+1Gtp78U
|
||||
0iTv8tdqyh8dPvmuF7UbGWMJEMMD5d2goEw2ZnkqmLPFK5jq8qAshaQw9wARAQAB
|
||||
tDdEaWdpdGFsIEFzc2V0IEhvbGRpbmdzLCBMTEMgPHNlY3VyaXR5QGRpZ2l0YWxh
|
||||
c3NldC5jb20+iQJOBBMBCAA4FiEE8m2KCq32ZsyyjyqxZQ7DJTtqj/UFAmO9khIC
|
||||
GwMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQZQ7DJTtqj/WMbg/+K0Mte9y+
|
||||
fCaWxFctfUbtd/JZBzpSCVMLN7PjZYZ50SwN/CqILUTFzzVLIx7uj/CyH/e1IV2O
|
||||
RR7mWFTSADmkdrM45RBCvDs2UEIl3Rpsg/4iRpCZo01YQL9Y1XyUid8F3cQYmwPk
|
||||
4YMY+tqqEhObAq0ngrGWiEWMUixbbRVqlPvRZDMeUNGdvmSOCs9LZLEnE9m4g2Kn
|
||||
lNKddfLZ+sHaq2bfOiB+mZECX6wTusjqQWeJPRdflVWwMxZ7IkG9YoQHGlg8fTMd
|
||||
3NqPE9OHOQiZhN4MbY6QZ70WexUNab8Pzf1Co4sSGhywVI3JibcqCNIbHW21+1py
|
||||
OItJvdMxeSscOde2Fm5Dqmhf8UE+xgvPXa5xA5Yf40AqwuKt7boGsMf09Lf7zitX
|
||||
5Zzl81saIPVC4OcM51t+sNDP6uJIynP5Dp1fxaIlb8gcQDqyWB/REr0vY1pRf/61
|
||||
M8+jfUP3RJMbX/tUiCxEG+1uDSGTqj2Ac4TqiXfFKpg+TdEzNFj9VtrzTJT/tIgj
|
||||
QlrKM9P9iB/JrNtqgeYrhaBZSpVKx4J7LNeIGdVJvRVzlW3tvCsTIT/lp/iJ1YjI
|
||||
FCdb76leR/PgQNdk4wyU4JLXOYueEPAbyiBqQwgmOoT8GpY1PP4dsFfu7MoV0Cq7
|
||||
//q+uwegRr5lLV6LwSBuFd1hqQ9ZdjAmmRi5Ag0EY72SEgEQAKP+D3bVJPC6sxSj
|
||||
q/3UH9hixNhcmG61w6X1uW0x5jMMYN72ilnDLbgsgA3qEyZ8G/i34nUU4K/WZkWg
|
||||
nJ59lOPIVf05yzEnesS6hbHXUzd6ayeWhPUzwxLBPy3yJUw7IRkFF9P9AMBaraAp
|
||||
27ZuWy40Ta8bVKc9DgEeWuesyFAqs74W7cRfGm0SCAp8R3I+Syoj66+jpXYJ7sFt
|
||||
eW4ITqrQcj64jBtGB8kQOe8JvC4COudXJ1BpKjExxIQlSK729tz0vsi+hzQfac/1
|
||||
m3j082sH89ZU8y4GQpjWo6YyEzIxKBgoEogD0CvYOeJ9nK1Uv3pVFKyC1KdysQ+h
|
||||
v+9V3zQoOaGF6115cIwQU1ewISUkiCOHzMYkrEXsbBOJlCmomuLnjMhsXht5tV4e
|
||||
c8axn6QM7qRfSR/3R0RZwdAca0oZBN4ZOokUuZnR7/FxyiOhKilGW5iX+0m1VvKH
|
||||
BImFM/VmCXw4hzcWZUZa5K6Ebpeg7zWN3a1kXZ+Kb2glqWYT5Pq3d1m+RtJOiuyn
|
||||
uyr1BnX6OvjTNWTmKPqO8x223dZpNGdK6sfUUeZ67OokI/l2dALOuZRcuCLK32LB
|
||||
uJmk/dLt4Bjem9ITFt2ECb1+RTa1aWOm8uS7BKUiDGedW6239h3HebdVenip1voY
|
||||
3EdwpiQxgsCD3g2Sbzj9M5UGOsWzABEBAAGJAjwEGAEIACYCGwwWIQTybYoKrfZm
|
||||
zLKPKrFlDsMlO2qP9QUCY72SxAUJA8JnsgAKCRBlDsMlO2qP9dfyD/9O76RZYI6w
|
||||
8xIEOoK/cw//4IA0bbN/vC2tn5l1zUba6TrXhCYKr96//YJS9Fd239Gf4kC7AEbS
|
||||
yf4ARLbezjtOVG33GlfrEFHfghMKhpjMQgb68NFw5U2eLMFc7BB/Fu4vSHqCMZ3I
|
||||
ajM/465kq+jLxTNiuI14MFs1OLGD5WbAo9VEzBUbi3mK/CB4xv2UEd2y6ZAZuCXO
|
||||
P2+Pr2P7W94ECu/N0dhnitkAirgXrS3nZSduLpjK/SkUzvdY642GHwy0i3M20Ztr
|
||||
p7o1Uu7ztlD9yDUbksMyhskG7I+k2NGLAwz/CG91GRrYdUpoWsPlU5XLyxjHCmSC
|
||||
q97qiRSKlGO3LbIiTRatrv+4fcdntN0EM/nJefdtKS8+qZqkPMGqURlDJcPnIpHk
|
||||
jGccrEJz4aGB0/4Kr9UDBnWDPsH92E6lRa5QlzDOolEqgFHyyRP1JYJH3RGKVlYK
|
||||
rcLlluADiRYXCadwtXvnkJGxfg2DGICn5bEInPtM+bEhO3IfqrjipvT/Qx3/N6T+
|
||||
hiHyl2Yyi0loUhbWsTuuSz+D07wj/4X1evuaaAc56RSwv0x6rLSjkYj1I7V3nMvc
|
||||
e2fwNFiJvLdGfMcIYyxrOwO24cFwzYMYoTDFmf8MkN/H/khKZiksdnIxfcBFfyWu
|
||||
PA8s5O3Zs90Ack3IvK7uAhRDz1PpR6Y+1bkCDQRjvZKEARAAuTgK6INJWBEzfrDM
|
||||
vM157ZGAM/7pyevj0WCDhqiCFdpH3MVt7+wq0tmR8Oo5Lt4AXqVtzn1bw1sMAkWK
|
||||
U6yxLtS7cMiXOAPOtemTzWQkvk9o1FFygRQ8oyp4RUP4wj+W4lYaDhY+tJRDr/sR
|
||||
6grYt/lZbfvEPuxL4jGW/dLSKHTLs8kh367Xm1qxqaG1C1tSLusTPb/8uNpOCANh
|
||||
A2HAJRCGMox7f295+mEWXujif8yIfYtSQldqh+2bA6vaV3WKtHTPdLa1zzB20rf9
|
||||
Mguz4ff3XDJCHPWOKeBOfqVS9CL67TZeOx0nJ6u2JnNDlwlzX7R63v1D/tSTYzPL
|
||||
mJeosIjpRQg4ELyyLSkj0lANvY/AwlKeTPkvoc76UwsQRFgxx6ZZjKObjAok6TQK
|
||||
HjszRNkeBWbbi8J+zvfS6U3+1qYtvf9Enpp1v1CWfEKZmC68MgspNCzLSOpkoAfe
|
||||
k2iQ/XsjKXSsaUXY5A1DljQTVbSs9G3OkQA0Eyv4JPj2KEXPoF/0sIt2QRrayyqk
|
||||
1lqN4k9a3zEZ2WpkQLIRK5DgCE/ORHXkperEWrDiAfSvuVl999jxr+Jqi8qvlPrm
|
||||
aQd0X5Wc5gpb7X72FMsb2UHaWsUEs6nwoAWnXgA3PGd0r9LihZMJXfMc+LSF/dRK
|
||||
fx+PizkTXQbfML8fi7Il9JA1p4UAEQEAAYkEcgQYAQgAJhYhBPJtigqt9mbMso8q
|
||||
sWUOwyU7ao/1BQJjvZKEAhsCBQkDwmcAAkAJEGUOwyU7ao/1wXQgBBkBCAAdFiEE
|
||||
ytw9HjtcTF+Upl14p79lqq27xJQFAmO9koQACgkQp79lqq27xJQG2hAAp4813NAu
|
||||
AOg4C/Yvq8aqnDRDHw/ISs5XsQTfVwbIssSiSTqdJb4jX0rbKW1qzM6l15EmEsPV
|
||||
5MCGfN8xfP5+UeeVIJaXLq3BMYJf1An8sun9f8Bp2Wdw6IDlr9VwFZ170JQ2xYvq
|
||||
VJ+s/rxbCJ8K9neDPelzN/KXMyUV/uA5D1G92IlItinw4ZqD9e/CjPfIBwfNEMnZ
|
||||
nYaku4VGJfzaMHezaUTB8UVyFVN6Zv2PGYEUBCwISM61IdnGKnJza0NMnEvGstXN
|
||||
vtnWk7H/12Q4/rDpApy68Qbuo8gbZIifjNY00u2iyx4BEvji418NfTdF5HuPHR4m
|
||||
g10cz+FcWxo13PGTXHKprNC9Y4M5nMAZW8z05/2geD8jzmY9Yz3m0GSVF/0cD3pB
|
||||
rQ/LXirxgJ2prCuE7Ax8XTTBg7+cjgqk0InKh2pF0sK+2UCbnN4hR+SQvR256hWI
|
||||
F+TP/rDryaqdubqCOh7kytPnPqZtL8VqK7yDRhfmgxv3+bpvm+B2qm1okUCkH3bb
|
||||
AkvowTBOcyTqLw7hYsREHkYVROYg57GGhMStkzaD+lep9kEUgcaXZF41W02WJeS3
|
||||
VYXwooxFBKMhzm+cluLV+ujC+FnRslh7q/u90+3N2VljEjxA4Oj3RNAARzpOs0V2
|
||||
BtuUsiPCTvhRLBmdG3RH25jm2hUPexP2+pMyEw//V211M6+MT5a8kCybK5e93I3+
|
||||
eT2bfAfd1k0kcQcfbocymxW5DJUqHgBj+G9ZC5PIAeFk+Jfld0y3M186NAvP8I4+
|
||||
ZNsJExdQyp1CN53mSWtxAadgHNNhDKX0KwyCarCk04xbf0qjlsrWNbsUI04sM1zt
|
||||
C46N/0JsCuG4uAztAfU9hjbLmSxpjf04Qqpc5NDlGLgZ2xQTVmXPlFg1DgrF6fIq
|
||||
WZwPa7z1eihkrEERPjnisjuwMd4uO5BIkqh8F7HdOnARYXpftg9LReV973z7i8n9
|
||||
4rhpBedAHwVRqWo8owM8DOVTaHAQzMnnzB+6nCoOcZc7PzhWtKKhZupW2DYaLdIh
|
||||
nlVCrmMSozkFn3shtOJ76XF2DMDpk0353w6i6rKghWC7TdpXPnWkHkExw4Pjnlse
|
||||
1NP2vdz183NKqEKros463i+hOszQj7jb5DiFxxOnKUfxBNEMJXTqYzXdEzw7Sncw
|
||||
NwTv4pFxnk3XFJD3IIXMdaCDYmHIJYK5Fwgc0Cop3dRAMJIB+0Q1/p+urDXqZphq
|
||||
AGroZ22Z1DXzv7rm1x2drZyOBohc+dqn3zjEx+lwZ6CY8XPiQgbWEzSzY8YT4oUA
|
||||
xRcs9cJ+0SK/HhW/EG51YNbr5IMDb3HvycHEreszEvwq2HdnsMIYdM8GC7fl7Zpp
|
||||
0r+S1089BYMqKmhepps=
|
||||
=srz3
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
PUB_KEY
|
||||
gpg_dir=$(mktemp -d)
|
||||
GNUPGHOME=$gpg_dir gpg --no-tty --quiet --import $key
|
||||
GNUPGHOME=$gpg_dir gpg --no-tty --quiet --command-fd 0 --edit-key F26D8A0AADF666CCB28F2AB1650EC3253B6A8FF5 << CMD
|
||||
trust
|
||||
4
|
||||
quit
|
||||
CMD
|
||||
GNUPGHOME=$gpg_dir gpg --verify $signature_file
|
||||
res=$?
|
||||
rm -rf $gpg_dir $key
|
||||
return $res
|
||||
}
|
||||
setvar() {
|
||||
echo "Setting '$1' to '$2'"
|
||||
echo "##vso[task.setvariable variable=$1;isOutput=true]$2"
|
||||
|
||||
158
ci/cron/check-releases.sh
Executable file
158
ci/cron/check-releases.sh
Executable file
@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
|
||||
|
||||
MAX_RELEASES=${MAX_RELEASES:-}
|
||||
AUTH=${AUTH:-"Authorization: token $GITHUB_TOKEN"}
|
||||
USER_AGENT="User-Agent: Daml cron (gary.verhaegen@digital-asset.com)"
|
||||
|
||||
export GNUPGHOME=$(mktemp -d)
|
||||
trap "rm -rf $GNUPGHOME" EXIT
|
||||
|
||||
setup_gpg() (
|
||||
key=$DIR/../pgp_pubkey
|
||||
log=$(mktemp)
|
||||
trap "rm -rf $log" EXIT
|
||||
if ! gpg --no-tty --quiet --import $key >>$log 2>&1; then
|
||||
echo "Failed to initialize GPG. Logs:"
|
||||
echo "---"
|
||||
cat $log
|
||||
echo "---"
|
||||
exit 1
|
||||
fi
|
||||
)
|
||||
|
||||
get_releases() (
|
||||
url="https://api.github.com/repos/digital-asset/daml/releases"
|
||||
tmp_dir=$(mktemp -d)
|
||||
trap "cd; rm -rf $tmp_dir" EXIT
|
||||
cd $tmp_dir
|
||||
while [ "$url" != "" ]; do
|
||||
curl $url \
|
||||
--fail \
|
||||
--silent \
|
||||
-H "$AUTH" \
|
||||
-H "$USER_AGENT" \
|
||||
-o >(cat - \
|
||||
| jq -c '.[]
|
||||
| { prerelease,
|
||||
tag: .tag_name[1:],
|
||||
assets: [.assets[] | .browser_download_url] }' \
|
||||
>> resp) \
|
||||
-D headers
|
||||
url=$(cat headers \
|
||||
| tr -d '\r' \
|
||||
| grep "link:" \
|
||||
| grep -Po '(?<=<)([^>]*)(?=>; rel="next")' \
|
||||
|| true)
|
||||
done
|
||||
cat resp
|
||||
)
|
||||
|
||||
retry() (
|
||||
attempts=$1
|
||||
cmd=$2
|
||||
cont=1
|
||||
delay=10
|
||||
while [ $cont == 1 ]; do
|
||||
if $cmd; then
|
||||
cont=0
|
||||
else
|
||||
echo " Exit $? for '$cmd', retrying."
|
||||
sleep $delay
|
||||
delay=$(( delay + delay ))
|
||||
attempts=$((attempts - 1))
|
||||
if [ "$attempts" = 0 ]; then
|
||||
echo "Max retries reached. Giving up on '$cmd'."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
done
|
||||
)
|
||||
|
||||
download_assets() (
|
||||
release=$1
|
||||
curl -H "$AUTH" \
|
||||
-H "$USER_AGENT" \
|
||||
--silent \
|
||||
--fail \
|
||||
--location \
|
||||
--parallel \
|
||||
--remote-name-all \
|
||||
$(echo $release | jq -r '.assets[]')
|
||||
)
|
||||
|
||||
verify_signatures() (
|
||||
log=$(mktemp)
|
||||
trap "rm -f $log" EXIT
|
||||
for f in $(ls | grep -v '\.asc$'); do
|
||||
if ! test -f $f.asc; then
|
||||
echo "No signature file on GitHub for $f."
|
||||
exit 1
|
||||
fi
|
||||
if ! gpg --verify $f.asc &>$log; then
|
||||
echo "Failed to verify signature for $f."
|
||||
echo "gpg logs:"
|
||||
echo "---"
|
||||
cat $log
|
||||
echo "---"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
)
|
||||
|
||||
verify_backup() (
|
||||
tag=$1
|
||||
gcs_base=gs://daml-data/releases/${tag#v}/github
|
||||
log=$(mktemp)
|
||||
trap "rm -f $log" EXIT
|
||||
for f in $(ls); do
|
||||
(
|
||||
if ! gsutil ls $gcs_base/$f &>/dev/null; then
|
||||
echo "No backup for $f; aborting."
|
||||
exit 1
|
||||
else
|
||||
gsutil cp $gcs_base/$f $f.gcs &>$log
|
||||
if ! diff $f $f.gcs; then
|
||||
echo "$f does not match backup; aborting."
|
||||
echo "gcs copy output:"
|
||||
echo "---"
|
||||
cat $log
|
||||
echo "---"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
) &
|
||||
done
|
||||
for pid in $(jobs -p); do
|
||||
wait $pid
|
||||
done
|
||||
)
|
||||
|
||||
check_release() (
|
||||
release=$1
|
||||
tag=$(echo $release | jq -r .tag)
|
||||
echo "[$(date --date=@$SECONDS -u +%H:%M:%S)] $tag"
|
||||
tmp_dir=$(mktemp -d)
|
||||
trap "cd; rm -rf $tmp_dir" EXIT
|
||||
cd $tmp_dir
|
||||
retry 5 "download_assets $release"
|
||||
verify_signatures
|
||||
verify_backup $tag
|
||||
)
|
||||
|
||||
setup_gpg
|
||||
releases=$(get_releases)
|
||||
|
||||
if [ "" != "$MAX_RELEASES" ]; then
|
||||
releases=$( (echo "$releases" | head -n $MAX_RELEASES) || test $? -eq 141)
|
||||
fi
|
||||
|
||||
for r in $releases; do
|
||||
check_release $r
|
||||
done
|
||||
@ -116,7 +116,7 @@ jobs:
|
||||
artifactName: perf-speedy
|
||||
|
||||
- job: check_releases
|
||||
timeoutInMinutes: 480
|
||||
timeoutInMinutes: 600
|
||||
pool:
|
||||
name: ubuntu_20_04
|
||||
demands: assignment -equals default
|
||||
@ -133,8 +133,9 @@ jobs:
|
||||
eval "$(dev-env/bin/dade assist)"
|
||||
source $(bash_lib)
|
||||
|
||||
bazel build //ci/cron:cron
|
||||
wrap_gcloud "$GCRED" "bazel-bin/ci/cron/cron check --bash-lib $(bash_lib)"
|
||||
export AUTH="$(get_gh_auth_header)"
|
||||
|
||||
wrap_gcloud "$GCRED" "ci/cron/check-releases.sh"
|
||||
displayName: check releases
|
||||
env:
|
||||
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|
||||
|
||||
@ -1,138 +0,0 @@
|
||||
-- Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
-- SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
module CheckReleases (check_releases) where
|
||||
|
||||
import Github
|
||||
|
||||
import qualified Control.Concurrent.Async
|
||||
import qualified Control.Concurrent.QSem
|
||||
import Control.Exception.Safe
|
||||
import qualified Control.Monad as Control
|
||||
import Control.Retry
|
||||
import Data.Conduit (runConduit, (.|))
|
||||
import Data.Conduit.Combinators (sinkHandle)
|
||||
import qualified Data.Foldable
|
||||
import Data.Maybe (isJust)
|
||||
import qualified Network.HTTP.Client as HTTP
|
||||
import Network.HTTP.Client.Conduit (bodyReaderSource)
|
||||
import qualified Network.HTTP.Client.TLS as TLS
|
||||
import qualified Network.URI
|
||||
import qualified System.Directory as Directory
|
||||
import qualified System.Exit as Exit
|
||||
import System.FilePath.Posix ((</>))
|
||||
import qualified System.IO.Extra as IO
|
||||
import qualified System.Process as System
|
||||
|
||||
shell :: String -> IO String
|
||||
shell cmd = System.readCreateProcess (System.shell cmd) ""
|
||||
|
||||
shell_ :: String -> IO ()
|
||||
shell_ cmd = Control.void $ shell cmd
|
||||
|
||||
download_assets :: FilePath -> GitHubRelease -> IO ()
|
||||
download_assets tmp release = do
|
||||
manager <- HTTP.newManager TLS.tlsManagerSettings
|
||||
tokens <- Control.Concurrent.QSem.newQSem 20
|
||||
Control.Concurrent.Async.forConcurrently_ (map uri $ assets release) $ \url ->
|
||||
bracket_
|
||||
(Control.Concurrent.QSem.waitQSem tokens)
|
||||
(Control.Concurrent.QSem.signalQSem tokens)
|
||||
(do
|
||||
req <- add_github_contact_header <$> HTTP.parseRequest (show url)
|
||||
recovering
|
||||
retryPolicy
|
||||
[retryHandler]
|
||||
(\_ -> downloadFile req manager url)
|
||||
)
|
||||
where -- Retry for 5 minutes total, doubling delay starting with 20ms
|
||||
retryPolicy = limitRetriesByCumulativeDelay (5 * 60 * 1000 * 1000) (exponentialBackoff (20 * 1000))
|
||||
retryHandler status =
|
||||
logRetries
|
||||
(\e -> pure $ isJust (fromException @IOException e) || isJust (fromException @HTTP.HttpException e)) -- Don’t try to be clever, just retry
|
||||
(\shouldRetry err status -> IO.hPutStrLn IO.stderr $ defaultLogMsg shouldRetry err status)
|
||||
status
|
||||
downloadFile req manager url = HTTP.withResponse req manager $ \resp -> do
|
||||
IO.withBinaryFile (tmp </> (last $ Network.URI.pathSegments url)) IO.WriteMode $ \handle ->
|
||||
runConduit $ bodyReaderSource (HTTP.responseBody resp) .| sinkHandle handle
|
||||
|
||||
verify_signatures :: FilePath -> FilePath -> String -> IO ()
|
||||
verify_signatures bash_lib tmp version_tag = do
|
||||
System.callCommand $ unlines ["bash -c '",
|
||||
"set -euo pipefail",
|
||||
"source \"" <> bash_lib <> "\"",
|
||||
"shopt -s extglob", -- enable !() pattern: things that _don't_ match
|
||||
"cd \"" <> tmp <> "\"",
|
||||
"for f in !(*.asc); do",
|
||||
"p=" <> version_tag <> "/github/$f",
|
||||
"if ! test -f $f.asc; then",
|
||||
"echo $p: no signature file",
|
||||
"else",
|
||||
"LOG=$(mktemp)",
|
||||
"if gpg_verify $f.asc >$LOG 2>&1; then",
|
||||
"echo $p: signature matches",
|
||||
"else",
|
||||
"echo $p: signature does not match",
|
||||
"echo Full gpg output:",
|
||||
"cat $LOG",
|
||||
"exit 2",
|
||||
"fi",
|
||||
"fi",
|
||||
"done",
|
||||
"'"]
|
||||
|
||||
does_backup_exist :: FilePath -> IO Bool
|
||||
does_backup_exist path = do
|
||||
out <- shell $ unlines ["bash -c '",
|
||||
"set -euo pipefail",
|
||||
"if gsutil ls \"" <> path <> "\" >/dev/null; then",
|
||||
"echo True",
|
||||
"else",
|
||||
"echo False",
|
||||
"fi",
|
||||
"'"]
|
||||
return $ read out
|
||||
|
||||
gcs_cp :: FilePath -> FilePath -> IO ()
|
||||
gcs_cp from to = do
|
||||
shell_ $ unlines ["bash -c '",
|
||||
"set -euo pipefail",
|
||||
"gsutil cp \"" <> from <> "\" \"" <> to <> "\" &>/dev/null",
|
||||
"'"]
|
||||
|
||||
check_files_match :: String -> String -> IO Bool
|
||||
check_files_match f1 f2 = do
|
||||
(exitCode, stdout, stderr) <- System.readProcessWithExitCode "diff" [f1, f2] ""
|
||||
case exitCode of
|
||||
Exit.ExitSuccess -> return True
|
||||
Exit.ExitFailure 1 -> return False
|
||||
Exit.ExitFailure _ -> fail $ "Diff failed.\n" ++ "STDOUT:\n" ++ stdout ++ "\nSTDERR:\n" ++ stderr
|
||||
|
||||
check_releases :: String -> Maybe Int -> IO ()
|
||||
check_releases bash_lib max_releases = do
|
||||
releases' <- fetch_gh_paginated "https://api.github.com/repos/digital-asset/daml/releases"
|
||||
let releases = case max_releases of
|
||||
Nothing -> releases'
|
||||
Just n -> take n releases'
|
||||
Data.Foldable.for_ releases (\release -> recoverAll retryPolicy $ \_ -> do
|
||||
let v = show $ tag release
|
||||
putStrLn $ "Checking release " <> v <> " ..."
|
||||
IO.withTempDir $ \temp_dir -> do
|
||||
download_assets temp_dir release
|
||||
verify_signatures bash_lib temp_dir v
|
||||
files <- Directory.listDirectory temp_dir
|
||||
Control.Concurrent.Async.forConcurrently_ files $ \f -> do
|
||||
let local_github = temp_dir </> f
|
||||
let local_gcp = temp_dir </> f <> ".gcp"
|
||||
let remote_gcp = "gs://daml-data/releases/" <> v <> "/github/" <> f
|
||||
exists <- does_backup_exist remote_gcp
|
||||
if exists then do
|
||||
gcs_cp remote_gcp local_gcp
|
||||
check_files_match local_github local_gcp >>= \case
|
||||
True -> putStrLn $ f <> " matches GCS backup."
|
||||
False -> fail $ f <> " does not match GCS backup."
|
||||
else do
|
||||
fail $ remote_gcp <> " does not exist. Aborting.")
|
||||
where
|
||||
-- Retry for 10 minutes total, delay of 1s
|
||||
retryPolicy = limitRetriesByCumulativeDelay (10 * 60 * 1000 * 1000) (constantDelay 1000_000)
|
||||
@ -1,83 +0,0 @@
|
||||
-- Copyright (c) 2023 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
-- SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
module Github
|
||||
( Asset(..)
|
||||
, GitHubRelease(..)
|
||||
, add_github_contact_header
|
||||
, fetch_gh_paginated
|
||||
) where
|
||||
|
||||
import Data.Aeson
|
||||
import qualified Data.ByteString.UTF8 as BS
|
||||
import qualified Data.CaseInsensitive as CI
|
||||
import Data.Function ((&))
|
||||
import Data.HashMap.Strict (HashMap)
|
||||
import qualified Data.HashMap.Strict as HMS
|
||||
import qualified Data.List.Split as Split
|
||||
import qualified Data.SemVer as SemVer
|
||||
import qualified Data.Text as Text
|
||||
import qualified Network.HTTP.Client as HTTP
|
||||
import qualified Network.HTTP.Client.TLS as TLS
|
||||
import Network.HTTP.Types.Status (statusCode)
|
||||
import Network.URI
|
||||
import qualified System.Exit as Exit
|
||||
import qualified Text.Regex.TDFA as Regex
|
||||
|
||||
data Asset = Asset { uri :: Network.URI.URI }
|
||||
instance FromJSON Asset where
|
||||
parseJSON = withObject "Asset" $ \v -> Asset
|
||||
<$> (do
|
||||
Just url <- Network.URI.parseURI <$> v .: "browser_download_url"
|
||||
return url)
|
||||
|
||||
data GitHubRelease = GitHubRelease { prerelease :: Bool, tag :: Version, assets :: [Asset] }
|
||||
instance FromJSON GitHubRelease where
|
||||
parseJSON = withObject "GitHubRelease" $ \v -> GitHubRelease
|
||||
<$> (v .: "prerelease")
|
||||
<*> (version . Text.tail <$> v .: "tag_name")
|
||||
<*> (v .: "assets")
|
||||
|
||||
data Version = Version SemVer.Version
|
||||
deriving (Ord, Eq)
|
||||
instance Show Version where
|
||||
show (Version v) = SemVer.toString v
|
||||
|
||||
version :: Text.Text -> Version
|
||||
version t = Version $ (\case Left s -> (error s); Right v -> v) $ SemVer.fromText t
|
||||
|
||||
fetch_gh_paginated :: String -> IO [GitHubRelease]
|
||||
fetch_gh_paginated url = do
|
||||
(resp_0, headers) <- http_get url
|
||||
case parse_next =<< HMS.lookup "link" headers of
|
||||
Nothing -> return resp_0
|
||||
Just next -> do
|
||||
rest <- fetch_gh_paginated next
|
||||
return $ resp_0 ++ rest
|
||||
where parse_next header = lookup "next" $ map parse_link $ split header
|
||||
split h = Split.splitOn ", " h
|
||||
link_regex = "<(.*)>; rel=\"(.*)\"" :: String
|
||||
parse_link l =
|
||||
let typed_regex :: (String, String, String, [String])
|
||||
typed_regex = l Regex.=~ link_regex
|
||||
in
|
||||
case typed_regex of
|
||||
(_, _, _, [url, rel]) -> (rel, url)
|
||||
_ -> error $ "Assumption violated: link header entry did not match regex.\nEntry: " <> l
|
||||
|
||||
http_get :: FromJSON a => String -> IO (a, HashMap String String)
|
||||
http_get url = do
|
||||
manager <- HTTP.newManager TLS.tlsManagerSettings
|
||||
request <- add_github_contact_header <$> HTTP.parseRequest url
|
||||
response <- HTTP.httpLbs request manager
|
||||
let body = decode $ HTTP.responseBody response
|
||||
let status = statusCode $ HTTP.responseStatus response
|
||||
case (status, body) of
|
||||
(200, Just body) -> return (body, response & HTTP.responseHeaders & map (\(n, v) -> (n & CI.foldedCase & BS.toString, BS.toString v)) & HMS.fromList)
|
||||
_ -> Exit.die $ unlines ["GET \"" <> url <> "\" returned status code " <> show status <> ".",
|
||||
show $ HTTP.responseBody response]
|
||||
|
||||
add_github_contact_header :: HTTP.Request -> HTTP.Request
|
||||
add_github_contact_header req =
|
||||
req { HTTP.requestHeaders = ("User-Agent", "Daml cron (team-daml-app-runtime@digitalasset.com)") : HTTP.requestHeaders req }
|
||||
|
||||
@ -4,29 +4,17 @@
|
||||
module Main (main) where
|
||||
|
||||
import qualified BazelCache
|
||||
import qualified CheckReleases
|
||||
|
||||
import qualified Control.Monad as Control
|
||||
import qualified Options.Applicative as Opt
|
||||
import qualified System.IO.Extra as IO
|
||||
|
||||
data CliArgs = Check { bash_lib :: String,
|
||||
max_releases :: Maybe Int }
|
||||
| BazelCache BazelCache.Opts
|
||||
data CliArgs = BazelCache BazelCache.Opts
|
||||
|
||||
parser :: Opt.ParserInfo CliArgs
|
||||
parser = info "This program is meant to be run by CI cron. You probably don't have sufficient access rights to run it locally."
|
||||
(Opt.hsubparser (Opt.command "check" check
|
||||
<> Opt.command "bazel-cache" bazelCache))
|
||||
(Opt.hsubparser (Opt.command "bazel-cache" bazelCache))
|
||||
where info t p = Opt.info (p Opt.<**> Opt.helper) (Opt.progDesc t)
|
||||
check = info "Check existing releases."
|
||||
(Check <$> Opt.strOption (Opt.long "bash-lib"
|
||||
<> Opt.metavar "PATH"
|
||||
<> Opt.help "Path to Bash library file.")
|
||||
<*> (Opt.optional $
|
||||
Opt.option Opt.auto (Opt.long "max-releases"
|
||||
<> Opt.metavar "INT"
|
||||
<> Opt.help "Max number of releases to check.")))
|
||||
bazelCache =
|
||||
info "Bazel cache debugging and fixing." $
|
||||
fmap BazelCache $ BazelCache.Opts
|
||||
@ -58,6 +46,4 @@ main = do
|
||||
\h -> IO.hSetBuffering h IO.LineBuffering
|
||||
opts <- Opt.execParser parser
|
||||
case opts of
|
||||
Check { bash_lib, max_releases } ->
|
||||
CheckReleases.check_releases bash_lib max_releases
|
||||
BazelCache opts -> BazelCache.run opts
|
||||
|
||||
87
ci/pgp_pubkey
Normal file
87
ci/pgp_pubkey
Normal file
@ -0,0 +1,87 @@
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQINBGO9khIBEAC/D5WTgMJQGQso1JfN5RTq6YiCBwJ+L84YfKCPUo1yW7/RQHNZ
|
||||
+5rYUQpGf1K5KCIhHtJeQyANzPy9KWnhDX6lIaoau6Dg9JK3SwNv20jDyCzZOjNW
|
||||
Gfajy7xVTWXmYM/us8/A5kJN4pwEGIUL73n2uOtOzhpJ6TGLujNKB5EfGUO1L2Jr
|
||||
v9BGx2ghv+dbdR3kPX6SYuj7U+tDvoaqJB8729kL14grpBqYy2YhF5eoLyvBaE9x
|
||||
brDydUCu5t2Xpr7yI7xGOhUSn2ygoP3e9YSjOhowj5U5oFtTGxvqSf7xd9gkFaZY
|
||||
uA58X3su0nxZ/9nbvb2RJPKtlUeOJS8pggXVSSGrHfWw3Bnu2G1pQNO+MYCS0Cu/
|
||||
gMxQTnJ4itUNoFb3c9dSnB/VXWxsvlK3F+EdFg9HLNiStJVxPhPwgTo138ohTI1H
|
||||
4eGdXpRPZSKNXGRRtWdbEseYBSDBzR0ulAn5TDXFDFjjJ5u7KJfdN7p9YaXWkXpB
|
||||
+hvsiWJuvUDxTGlQE02PQjyN5vzj1NaU7CRRLvOYSstsOyTmuYg/xxvqA9XbPdti
|
||||
g9AtaeYSjRzq7OBq79FhcmKDOfh7Zc07RRXHy2xTdvw+Iy5HEjk0fYFz+1Gtp78U
|
||||
0iTv8tdqyh8dPvmuF7UbGWMJEMMD5d2goEw2ZnkqmLPFK5jq8qAshaQw9wARAQAB
|
||||
tDdEaWdpdGFsIEFzc2V0IEhvbGRpbmdzLCBMTEMgPHNlY3VyaXR5QGRpZ2l0YWxh
|
||||
c3NldC5jb20+iQJOBBMBCAA4FiEE8m2KCq32ZsyyjyqxZQ7DJTtqj/UFAmO9khIC
|
||||
GwMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQZQ7DJTtqj/WMbg/+K0Mte9y+
|
||||
fCaWxFctfUbtd/JZBzpSCVMLN7PjZYZ50SwN/CqILUTFzzVLIx7uj/CyH/e1IV2O
|
||||
RR7mWFTSADmkdrM45RBCvDs2UEIl3Rpsg/4iRpCZo01YQL9Y1XyUid8F3cQYmwPk
|
||||
4YMY+tqqEhObAq0ngrGWiEWMUixbbRVqlPvRZDMeUNGdvmSOCs9LZLEnE9m4g2Kn
|
||||
lNKddfLZ+sHaq2bfOiB+mZECX6wTusjqQWeJPRdflVWwMxZ7IkG9YoQHGlg8fTMd
|
||||
3NqPE9OHOQiZhN4MbY6QZ70WexUNab8Pzf1Co4sSGhywVI3JibcqCNIbHW21+1py
|
||||
OItJvdMxeSscOde2Fm5Dqmhf8UE+xgvPXa5xA5Yf40AqwuKt7boGsMf09Lf7zitX
|
||||
5Zzl81saIPVC4OcM51t+sNDP6uJIynP5Dp1fxaIlb8gcQDqyWB/REr0vY1pRf/61
|
||||
M8+jfUP3RJMbX/tUiCxEG+1uDSGTqj2Ac4TqiXfFKpg+TdEzNFj9VtrzTJT/tIgj
|
||||
QlrKM9P9iB/JrNtqgeYrhaBZSpVKx4J7LNeIGdVJvRVzlW3tvCsTIT/lp/iJ1YjI
|
||||
FCdb76leR/PgQNdk4wyU4JLXOYueEPAbyiBqQwgmOoT8GpY1PP4dsFfu7MoV0Cq7
|
||||
//q+uwegRr5lLV6LwSBuFd1hqQ9ZdjAmmRi5Ag0EY72SEgEQAKP+D3bVJPC6sxSj
|
||||
q/3UH9hixNhcmG61w6X1uW0x5jMMYN72ilnDLbgsgA3qEyZ8G/i34nUU4K/WZkWg
|
||||
nJ59lOPIVf05yzEnesS6hbHXUzd6ayeWhPUzwxLBPy3yJUw7IRkFF9P9AMBaraAp
|
||||
27ZuWy40Ta8bVKc9DgEeWuesyFAqs74W7cRfGm0SCAp8R3I+Syoj66+jpXYJ7sFt
|
||||
eW4ITqrQcj64jBtGB8kQOe8JvC4COudXJ1BpKjExxIQlSK729tz0vsi+hzQfac/1
|
||||
m3j082sH89ZU8y4GQpjWo6YyEzIxKBgoEogD0CvYOeJ9nK1Uv3pVFKyC1KdysQ+h
|
||||
v+9V3zQoOaGF6115cIwQU1ewISUkiCOHzMYkrEXsbBOJlCmomuLnjMhsXht5tV4e
|
||||
c8axn6QM7qRfSR/3R0RZwdAca0oZBN4ZOokUuZnR7/FxyiOhKilGW5iX+0m1VvKH
|
||||
BImFM/VmCXw4hzcWZUZa5K6Ebpeg7zWN3a1kXZ+Kb2glqWYT5Pq3d1m+RtJOiuyn
|
||||
uyr1BnX6OvjTNWTmKPqO8x223dZpNGdK6sfUUeZ67OokI/l2dALOuZRcuCLK32LB
|
||||
uJmk/dLt4Bjem9ITFt2ECb1+RTa1aWOm8uS7BKUiDGedW6239h3HebdVenip1voY
|
||||
3EdwpiQxgsCD3g2Sbzj9M5UGOsWzABEBAAGJAjwEGAEIACYCGwwWIQTybYoKrfZm
|
||||
zLKPKrFlDsMlO2qP9QUCY72SxAUJA8JnsgAKCRBlDsMlO2qP9dfyD/9O76RZYI6w
|
||||
8xIEOoK/cw//4IA0bbN/vC2tn5l1zUba6TrXhCYKr96//YJS9Fd239Gf4kC7AEbS
|
||||
yf4ARLbezjtOVG33GlfrEFHfghMKhpjMQgb68NFw5U2eLMFc7BB/Fu4vSHqCMZ3I
|
||||
ajM/465kq+jLxTNiuI14MFs1OLGD5WbAo9VEzBUbi3mK/CB4xv2UEd2y6ZAZuCXO
|
||||
P2+Pr2P7W94ECu/N0dhnitkAirgXrS3nZSduLpjK/SkUzvdY642GHwy0i3M20Ztr
|
||||
p7o1Uu7ztlD9yDUbksMyhskG7I+k2NGLAwz/CG91GRrYdUpoWsPlU5XLyxjHCmSC
|
||||
q97qiRSKlGO3LbIiTRatrv+4fcdntN0EM/nJefdtKS8+qZqkPMGqURlDJcPnIpHk
|
||||
jGccrEJz4aGB0/4Kr9UDBnWDPsH92E6lRa5QlzDOolEqgFHyyRP1JYJH3RGKVlYK
|
||||
rcLlluADiRYXCadwtXvnkJGxfg2DGICn5bEInPtM+bEhO3IfqrjipvT/Qx3/N6T+
|
||||
hiHyl2Yyi0loUhbWsTuuSz+D07wj/4X1evuaaAc56RSwv0x6rLSjkYj1I7V3nMvc
|
||||
e2fwNFiJvLdGfMcIYyxrOwO24cFwzYMYoTDFmf8MkN/H/khKZiksdnIxfcBFfyWu
|
||||
PA8s5O3Zs90Ack3IvK7uAhRDz1PpR6Y+1bkCDQRjvZKEARAAuTgK6INJWBEzfrDM
|
||||
vM157ZGAM/7pyevj0WCDhqiCFdpH3MVt7+wq0tmR8Oo5Lt4AXqVtzn1bw1sMAkWK
|
||||
U6yxLtS7cMiXOAPOtemTzWQkvk9o1FFygRQ8oyp4RUP4wj+W4lYaDhY+tJRDr/sR
|
||||
6grYt/lZbfvEPuxL4jGW/dLSKHTLs8kh367Xm1qxqaG1C1tSLusTPb/8uNpOCANh
|
||||
A2HAJRCGMox7f295+mEWXujif8yIfYtSQldqh+2bA6vaV3WKtHTPdLa1zzB20rf9
|
||||
Mguz4ff3XDJCHPWOKeBOfqVS9CL67TZeOx0nJ6u2JnNDlwlzX7R63v1D/tSTYzPL
|
||||
mJeosIjpRQg4ELyyLSkj0lANvY/AwlKeTPkvoc76UwsQRFgxx6ZZjKObjAok6TQK
|
||||
HjszRNkeBWbbi8J+zvfS6U3+1qYtvf9Enpp1v1CWfEKZmC68MgspNCzLSOpkoAfe
|
||||
k2iQ/XsjKXSsaUXY5A1DljQTVbSs9G3OkQA0Eyv4JPj2KEXPoF/0sIt2QRrayyqk
|
||||
1lqN4k9a3zEZ2WpkQLIRK5DgCE/ORHXkperEWrDiAfSvuVl999jxr+Jqi8qvlPrm
|
||||
aQd0X5Wc5gpb7X72FMsb2UHaWsUEs6nwoAWnXgA3PGd0r9LihZMJXfMc+LSF/dRK
|
||||
fx+PizkTXQbfML8fi7Il9JA1p4UAEQEAAYkEcgQYAQgAJhYhBPJtigqt9mbMso8q
|
||||
sWUOwyU7ao/1BQJjvZKEAhsCBQkDwmcAAkAJEGUOwyU7ao/1wXQgBBkBCAAdFiEE
|
||||
ytw9HjtcTF+Upl14p79lqq27xJQFAmO9koQACgkQp79lqq27xJQG2hAAp4813NAu
|
||||
AOg4C/Yvq8aqnDRDHw/ISs5XsQTfVwbIssSiSTqdJb4jX0rbKW1qzM6l15EmEsPV
|
||||
5MCGfN8xfP5+UeeVIJaXLq3BMYJf1An8sun9f8Bp2Wdw6IDlr9VwFZ170JQ2xYvq
|
||||
VJ+s/rxbCJ8K9neDPelzN/KXMyUV/uA5D1G92IlItinw4ZqD9e/CjPfIBwfNEMnZ
|
||||
nYaku4VGJfzaMHezaUTB8UVyFVN6Zv2PGYEUBCwISM61IdnGKnJza0NMnEvGstXN
|
||||
vtnWk7H/12Q4/rDpApy68Qbuo8gbZIifjNY00u2iyx4BEvji418NfTdF5HuPHR4m
|
||||
g10cz+FcWxo13PGTXHKprNC9Y4M5nMAZW8z05/2geD8jzmY9Yz3m0GSVF/0cD3pB
|
||||
rQ/LXirxgJ2prCuE7Ax8XTTBg7+cjgqk0InKh2pF0sK+2UCbnN4hR+SQvR256hWI
|
||||
F+TP/rDryaqdubqCOh7kytPnPqZtL8VqK7yDRhfmgxv3+bpvm+B2qm1okUCkH3bb
|
||||
AkvowTBOcyTqLw7hYsREHkYVROYg57GGhMStkzaD+lep9kEUgcaXZF41W02WJeS3
|
||||
VYXwooxFBKMhzm+cluLV+ujC+FnRslh7q/u90+3N2VljEjxA4Oj3RNAARzpOs0V2
|
||||
BtuUsiPCTvhRLBmdG3RH25jm2hUPexP2+pMyEw//V211M6+MT5a8kCybK5e93I3+
|
||||
eT2bfAfd1k0kcQcfbocymxW5DJUqHgBj+G9ZC5PIAeFk+Jfld0y3M186NAvP8I4+
|
||||
ZNsJExdQyp1CN53mSWtxAadgHNNhDKX0KwyCarCk04xbf0qjlsrWNbsUI04sM1zt
|
||||
C46N/0JsCuG4uAztAfU9hjbLmSxpjf04Qqpc5NDlGLgZ2xQTVmXPlFg1DgrF6fIq
|
||||
WZwPa7z1eihkrEERPjnisjuwMd4uO5BIkqh8F7HdOnARYXpftg9LReV973z7i8n9
|
||||
4rhpBedAHwVRqWo8owM8DOVTaHAQzMnnzB+6nCoOcZc7PzhWtKKhZupW2DYaLdIh
|
||||
nlVCrmMSozkFn3shtOJ76XF2DMDpk0353w6i6rKghWC7TdpXPnWkHkExw4Pjnlse
|
||||
1NP2vdz183NKqEKros463i+hOszQj7jb5DiFxxOnKUfxBNEMJXTqYzXdEzw7Sncw
|
||||
NwTv4pFxnk3XFJD3IIXMdaCDYmHIJYK5Fwgc0Cop3dRAMJIB+0Q1/p+urDXqZphq
|
||||
AGroZ22Z1DXzv7rm1x2drZyOBohc+dqn3zjEx+lwZ6CY8XPiQgbWEzSzY8YT4oUA
|
||||
xRcs9cJ+0SK/HhW/EG51YNbr5IMDb3HvycHEreszEvwq2HdnsMIYdM8GC7fl7Zpp
|
||||
0r+S1089BYMqKmhepps=
|
||||
=srz3
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
Loading…
Reference in New Issue
Block a user