ci/cron: do not push artifacts to gcs bucket (#8067)

Having the cron push artifacts to GCP was really only meant to happen
once. I got distracted and worked on other things. This PR closes that
work loop such that the current state and expectations are:

- Every new release pushes to GCP as part of the release process.
- The cron only checks that the GCP backup exists and matches, but does
  not push if it doesn't.

The reason for this is we want the cron job to fail if there are
additional, unexpected files in a release, rather than automatically
commit those files for the long term.

CHANGELOG_BEGIN
CHANGELOG_END
This commit is contained in:
Gary Verhaegen 2020-11-25 19:12:03 +01:00 committed by GitHub
parent 06bec6a591
commit 3cef53135c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 20 additions and 9 deletions

View File

@ -175,7 +175,7 @@ jobs:
GCRED: $(GOOGLE_APPLICATION_CREDENTIALS_CONTENT)
- job: check_releases
timeoutInMinutes: 120
timeoutInMinutes: 240
pool:
name: linux-pool
demands: assignment -equals default

View File

@ -336,8 +336,8 @@ does_backup_exist gcp_credentials bash_lib path = do
"'"]
return $ read out
push_to_gcp :: String -> FilePath -> FilePath -> FilePath -> IO ()
push_to_gcp gcp_credentials bash_lib local_path remote_path = do
gcs_cp :: String -> FilePath -> FilePath -> FilePath -> IO ()
gcs_cp gcp_credentials bash_lib local_path remote_path = do
shell_ $ unlines ["bash -c '",
"set -euo pipefail",
"eval \"$(dev-env/bin/dade assist)\"",
@ -349,6 +349,14 @@ push_to_gcp gcp_credentials bash_lib local_path remote_path = do
"gcs \"$GCRED\" cp \"" <> local_path <> "\" \"" <> remote_path <> "\"",
"'"]
check_files_match :: String -> String -> IO Bool
check_files_match f1 f2 = do
(exitCode, stdout, stderr) <- System.readProcessWithExitCode "diff" [f1, f2] ""
case exitCode of
Exit.ExitSuccess -> return True
Exit.ExitFailure 1 -> return False
Exit.ExitFailure _ -> fail $ "Diff failed.\n" ++ "STDOUT:\n" ++ stdout ++ "\nSTDERR:\n" ++ stderr
check_releases :: Maybe String -> String -> Maybe Int -> IO ()
check_releases gcp_credentials bash_lib max_releases = do
releases' <- fetch_gh_paginated "https://api.github.com/repos/digital-asset/daml/releases"
@ -363,14 +371,17 @@ check_releases gcp_credentials bash_lib max_releases = do
verify_signatures bash_lib temp_dir v >>= putStrLn
Control.Monad.Extra.whenJust gcp_credentials $ \gcred ->
Directory.listDirectory temp_dir >>= Data.Foldable.traverse_ (\f -> do
let gcp_path = "gs://daml-data/releases/" <> v <> "/github/" <> f
exists <- does_backup_exist gcred bash_lib gcp_path
let local_github = temp_dir </> f
let local_gcp = temp_dir </> f <> ".gcp"
let remote_gcp = "gs://daml-data/releases/" <> v <> "/github/" <> f
exists <- does_backup_exist gcred bash_lib remote_gcp
if exists then do
putStrLn $ gcp_path <> " already exists."
gcs_cp gcred bash_lib remote_gcp local_gcp
check_files_match local_github local_gcp >>= \case
True -> putStrLn $ f <> " matches GCS backup."
False -> Exit.die $ f <> " does not match GCS backup."
else do
putStr $ gcp_path <> " does not exist; pushing..."
push_to_gcp gcred bash_lib (temp_dir </> f) gcp_path
putStrLn " done."))
Exit.die $ remote_gcp <> " does not exist. Aborting."))
data CliArgs = Docs
| Check { bash_lib :: String,