mirror of
https://github.com/digital-asset/daml.git
synced 2024-08-17 00:30:25 +03:00
This commit aims at mitigating two issues we have noticed with the 0.13.41 release: 1. The initial cron run for that release got interrupted at the 50 minutes mark, which happened to be right in the middle of the s3 upload. This means it had already changed the versions.json file, but had not finished updating the actual html files. Right now, the docs.daml.com website shows version 0.13.41 in the drop-down, but actually displays the content for 0.13.40. Additionally, trying to explicitly visit the website for 0.13.41 (https://docs.daml.com/0.13.41) yields a 404. Note that this also means the cron job did not reach the "tell HubSpot" point, so 0.13.41 did not get announced. 2. As the script also did not reach the "clear cache" step, subsequent runs have been rebuilding the documentation for no reason as the sequence of steps was: check versions.json through HTTP, get cached one, see it's not up-to-date, build docs, check versions.json through s3 API, bypassing the cache, see it's up-to-date, stop. To address those issues, this PR changes the cron to: 1. Increase the timeout to 2h instead of 50 minutes. 2. Always check the versions.json file through s3, rather than go through the HTTP cache first. These are not complete solutions but I'm not sure how to do better given that s3 does not have atomic operations.
146 lines
5.6 KiB
YAML
146 lines
5.6 KiB
YAML
# Azure Pipelines file, see https://aka.ms/yaml
|
|
|
|
# Do not run on PRs
|
|
pr: none
|
|
|
|
# Do not run on merge to master
|
|
trigger: none
|
|
|
|
# Do run on a schedule (hourly)
|
|
#
|
|
# This is currently (2019-08-15) broken on Azure for GitHub-hosted repos. It
|
|
# does, however, work as expected for Azure-hosted repos. As a workaround, we
|
|
# have created a repo inside Azure that contains an `azure-pipelines.yml` file
|
|
# that just triggers this job.
|
|
#
|
|
# When the situation is resolved, delete that repo in Azure and uncomment the
|
|
# following. In the meantime, this should stay commented so we avoid running
|
|
# jobs twice when Azure fixes this issue.
|
|
#schedules:
|
|
#- cron: "0 * * * *"
|
|
# displayName: hourly cron
|
|
# branches:
|
|
# include:
|
|
# - master
|
|
# always: true
|
|
|
|
jobs:
|
|
- job: docs
|
|
timeoutInMinutes: 120
|
|
pool:
|
|
name: 'linux-pool'
|
|
steps:
|
|
- checkout: self
|
|
- bash: ci/dev-env-install.sh
|
|
displayName: 'Build/Install the Developer Environment'
|
|
- bash: ci/cron/run.sh
|
|
env:
|
|
AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
|
|
AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
|
|
HUBSPOT_TOKEN: $(HUBSPOT_TOKEN)
|
|
- bash: |
|
|
set -euo pipefail
|
|
MESSAGE=$(git log --pretty=format:%s -n1)
|
|
curl -XPOST \
|
|
-i \
|
|
-H 'Content-type: application/json' \
|
|
--data "{\"text\":\"<!here> *FAILED* Daily Docs: <https://dev.azure.com/digitalasset/daml/_build/results?buildId=$(Build.BuildId)|$MESSAGE>\n\"}" \
|
|
$(Slack.URL)
|
|
condition: and(failed(), eq(variables['Build.SourceBranchName'], 'master'))
|
|
|
|
- job: docker_image
|
|
timeoutInMinutes: 60
|
|
pool:
|
|
name: 'linux-pool'
|
|
steps:
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
eval "$(dev-env/bin/dade-assist)"
|
|
echo $DOCKER_PASSWORD | docker login --username $DOCKER_LOGIN --password-stdin
|
|
RELEASES=$(curl https://api.github.com/repos/digital-asset/daml/releases -s | jq -r '. | map(select(.prerelease == false)) | map(.tag_name)[]')
|
|
DIR=$(pwd)
|
|
VERSIONS=$(curl 'https://hub.docker.com/v2/repositories/digitalasset/daml-sdk/tags/?page_size=10000' -s)
|
|
DOCKERFILE_CHANGE_TIME=$(date -uIs -d $(git log -1 --format="%cI" -- ci/docker/daml-sdk/Dockerfile))
|
|
for version in $(echo $RELEASES | sed -e 's/ /\n/g'); do
|
|
LAST_UPDATE=$(echo $VERSIONS | jq -r '.results[] | select(.name == "'${version#v}'") | .last_updated')
|
|
if [[ -n "$LAST_UPDATE" && "$LAST_UPDATE" > "$DOCKERFILE_CHANGE_TIME" ]]; then
|
|
echo "${version#v} already exists and is up to date, skipping."
|
|
else
|
|
if [[ -n "$LAST_UPDATE" ]]; then
|
|
echo "${version#v} already exists but Dockerfile has changed; rebuilding..."
|
|
fi
|
|
echo "Building version ${version#v}..."
|
|
cd ci/docker/daml-sdk
|
|
docker build -t digitalasset/daml-sdk:${version#v} --build-arg VERSION=${version#v} .
|
|
docker push digitalasset/daml-sdk:${version#v}
|
|
cd "$DIR"
|
|
echo "Done."
|
|
fi
|
|
done
|
|
env:
|
|
DOCKER_LOGIN: $(DOCKER_LOGIN)
|
|
DOCKER_PASSWORD: $(DOCKER_PASSWORD)
|
|
|
|
- job: vscode_marketplace
|
|
timeoutInMinutes: 10
|
|
pool:
|
|
name: 'linux-pool'
|
|
steps:
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
eval "$(dev-env/bin/dade-assist)"
|
|
|
|
AUTH=$(echo -n "OAuth:${MARKETPLACE_TOKEN}" | base64 -w0)
|
|
MARKET=$(curl -H "Authorization: Basic $AUTH" \
|
|
-H "Accept: application/json;api-version=5.0-preview.2" \
|
|
-s \
|
|
"https://marketplace.visualstudio.com/_apis/gallery/publishers/DigitalAssetHoldingsLLC/extensions/daml?flags=1" \
|
|
| jq -r '.versions[0].version')
|
|
GITHUB=$(curl https://api.github.com/repos/digital-asset/daml/releases -s | jq -r '. | map(select(.prerelease == false)) | map(.tag_name)[0]')
|
|
if [[ "${GITHUB#v}" != "$MARKET" ]] && git merge-base --is-ancestor 798e96c9b9034eac85ace786b9e1955cf380285c $GITHUB; then
|
|
echo "Publishing $GITHUB to VSCode Marketplace"
|
|
git checkout $GITHUB
|
|
cd compiler/daml-extension
|
|
# This produces out/src/extension.js
|
|
bazel run @nodejs//:bin/yarn
|
|
bazel run @nodejs//:bin/yarn compile
|
|
bazel run --run_under="cd $PWD && " @daml_extension_deps//vsce/bin:vsce -- publish ${GITHUB#v} -p $MARKETPLACE_TOKEN
|
|
else
|
|
if [[ "${GITHUB#v}" == "$MARKET" ]]; then
|
|
echo "Version on marketplace is already the latest ($GITHUB)."
|
|
else
|
|
echo "Latest version is not ready for marketplace publication."
|
|
fi
|
|
fi
|
|
env:
|
|
MARKETPLACE_TOKEN: $(VSCODE_MARKETPLACE_TOKEN)
|
|
|
|
- job: download_stats
|
|
timeoutInMinutes: 10
|
|
pool:
|
|
name: "linux-pool"
|
|
steps:
|
|
- checkout: self
|
|
- bash: |
|
|
set -euo pipefail
|
|
|
|
eval "$(dev-env/bin/dade-assist)"
|
|
|
|
STATS=$(mktemp)
|
|
curl https://api.github.com/repos/digital-asset/daml/releases -s | gzip -9 > $STATS
|
|
|
|
GCS_KEY=$(mktemp)
|
|
cleanup () {
|
|
rm -f $GCS_KEY
|
|
}
|
|
trap cleanup EXIT
|
|
echo "$GOOGLE_APPLICATION_CREDENTIALS_CONTENT" > $GCS_KEY
|
|
gcloud auth activate-service-account --key-file=$GCS_KEY
|
|
BOTO_CONFIG=/dev/null gsutil cp $STATS gs://daml-data/downloads/$(date -u +%Y%m%d_%H%M%SZ).json.gz
|
|
env:
|
|
GOOGLE_APPLICATION_CREDENTIALS_CONTENT: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
|