Prune SDK versions in compat tests (#6325)

This PR does two things:

1. It makes use of the new sha256sums file to speed up the script to
   update the versions significantly. The code here is a bit hacky but
   as mentioned in the comment, this script is only run offline atm so
   that seems reasonable.
2. It prunes SDK snapshot versions to those for which there is no
   corresponding stable release.

changelog_begin
changelog_end
This commit is contained in:
Moritz Kiefer 2020-06-12 17:03:38 +02:00 committed by GitHub
parent 5329a4f97f
commit ccd36146a4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 70 additions and 164 deletions

View File

@ -248,7 +248,7 @@ daml_sdk_head(
[
daml_sdk(
# See the explanation in UpdateVersions.hs
create_daml_app_patch = None if version_sha256s.get(ver).get("create_daml_app_patch") else "@daml-sdk-1.1.0-snapshot.20200422.3991.0.6391ee9f//:create_daml_app.patch",
create_daml_app_patch = None if version_sha256s.get(ver).get("create_daml_app_patch") else "@daml-sdk-1.1.1//:create_daml_app.patch",
create_daml_app_patch_sha256 = version_sha256s.get(ver).get("create_daml_app_patch"),
daml_ledger_sha256 = version_sha256s.get(ver).get("daml_ledger"),
daml_react_sha256 = version_sha256s.get(ver).get("daml_react"),

View File

@ -32,32 +32,6 @@ excluded_test_tool_tests = {
"exclusions": ["ContractKeysIT", "ContractKeysSubmitterIsMaintainerIT"],
},
],
"1.0.1-snapshot.20200417.3908.1.722bac90": [
{
"start": "1.0.1-snapshot.20200424.3917.0.16093690",
"end": "1.0.1",
"exclusions": ["ContractKeysIT"],
},
{
"start": "1.1.0-snapshot.20200430.4057.0.681c862d",
"exclusions": ["ContractKeysIT", "ContractKeysSubmitterIsMaintainerIT"],
},
],
"1.0.1-snapshot.20200424.3917.0.16093690": [
{
"end": "1.0.1-snapshot.20200417.3908.1.722bac90",
"exclusions": ["ContractKeysIT"],
},
{
"start": "1.1.0-snapshot.20200422.3991.0.6391ee9f",
"end": "1.1.0-snapshot.20200422.3991.0.6391ee9f",
"exclusions": ["ContractKeysIT"],
},
{
"start": "1.1.0-snapshot.20200430.4057.0.681c862d",
"exclusions": ["ContractKeysSubmitterIsMaintainerIT"],
},
],
"1.0.1": [
{
"end": "1.0.1-snapshot.20200417.3908.1.722bac90",
@ -73,39 +47,6 @@ excluded_test_tool_tests = {
"exclusions": ["ContractKeysSubmitterIsMaintainerIT"],
},
],
"1.1.0-snapshot.20200422.3991.0.6391ee9f": [
{
"start": "1.0.1-snapshot.20200424.3917.0.16093690",
"end": "1.0.1",
"exclusions": ["ContractKeysIT"],
},
{
"start": "1.1.0-snapshot.20200430.4057.0.681c862d",
"exclusions": ["ContractKeysIT", "ContractKeysSubmitterIsMaintainerIT"],
},
],
"1.1.0-snapshot.20200430.4057.0.681c862d": [
{
"end": "1.0.1-snapshot.20200417.3908.1.722bac90",
"exclusions": ["ContractKeysIT"],
},
{
"start": "1.1.0-snapshot.20200422.3991.0.6391ee9f",
"end": "1.1.0-snapshot.20200422.3991.0.6391ee9f",
"exclusions": ["ContractKeysIT"],
},
],
"1.1.0-snapshot.20200506.4107.0.7e448d81": [
{
"end": "1.0.1-snapshot.20200417.3908.1.722bac90",
"exclusions": ["ContractKeysIT"],
},
{
"start": "1.1.0-snapshot.20200422.3991.0.6391ee9f",
"end": "1.1.0-snapshot.20200422.3991.0.6391ee9f",
"exclusions": ["ContractKeysIT"],
},
],
"1.1.1": [
{
"end": "1.0.1-snapshot.20200417.3908.1.722bac90",
@ -117,7 +58,7 @@ excluded_test_tool_tests = {
"exclusions": ["ContractKeysIT"],
},
],
"1.2.0-snapshot.20200513.4172.0.021f4af3": [
"1.2.0": [
{
"end": "1.0.1-snapshot.20200417.3908.1.722bac90",
"exclusions": ["ContractKeysIT"],
@ -128,7 +69,7 @@ excluded_test_tool_tests = {
"exclusions": ["ContractKeysIT"],
},
],
"1.2.0-snapshot.20200520.4224.0.2af134ca": [
"1.3.0-snapshot.20200603.4345.0.1386abc0": [
{
"end": "1.0.1-snapshot.20200417.3908.1.722bac90",
"exclusions": ["ContractKeysIT"],
@ -139,7 +80,7 @@ excluded_test_tool_tests = {
"exclusions": ["ContractKeysIT"],
},
],
"1.2.0-snapshot.20200520.4228.0.595f1e27": [
"1.3.0-snapshot.20200610.4412.0.0544323d": [
{
"end": "1.0.1-snapshot.20200417.3908.1.722bac90",
"exclusions": ["ContractKeysIT"],

View File

@ -4,39 +4,30 @@
# Update versions/UpdateVersions.hs instead.
sdk_versions = [
"1.0.0",
"1.0.1-snapshot.20200417.3908.1.722bac90",
"1.0.1-snapshot.20200424.3917.0.16093690",
"1.0.1",
"1.1.0-snapshot.20200422.3991.0.6391ee9f",
"1.1.0-snapshot.20200430.4057.0.681c862d",
"1.1.0-snapshot.20200506.4107.0.7e448d81",
"1.1.1",
"1.2.0-snapshot.20200513.4172.0.021f4af3",
"1.2.0-snapshot.20200520.4224.0.2af134ca",
"1.2.0-snapshot.20200520.4228.0.595f1e27",
"1.2.0",
"1.3.0-snapshot.20200603.4345.0.1386abc0",
"1.3.0-snapshot.20200610.4412.0.0544323d",
"0.0.0",
]
platform_versions = [
"1.0.0",
"1.0.1-snapshot.20200417.3908.1.722bac90",
"1.0.1-snapshot.20200424.3917.0.16093690",
"1.0.1",
"1.1.0-snapshot.20200422.3991.0.6391ee9f",
"1.1.0-snapshot.20200430.4057.0.681c862d",
"1.1.0-snapshot.20200506.4107.0.7e448d81",
"1.1.1",
"1.2.0-snapshot.20200513.4172.0.021f4af3",
"1.2.0-snapshot.20200520.4224.0.2af134ca",
"1.2.0-snapshot.20200520.4228.0.595f1e27",
"1.2.0",
"1.3.0-snapshot.20200603.4345.0.1386abc0",
"1.3.0-snapshot.20200610.4412.0.0544323d",
"0.0.0",
]
stable_versions = [
"1.0.0",
"1.0.1",
"1.1.1",
"1.2.0",
"0.0.0",
]
latest_stable_version = "1.1.1"
latest_stable_version = "1.2.0"
version_sha256s = {
"1.0.0": {
"linux": "ee7e2f50394d44fb3490068de64d37f4f87534e802717bd7e07e8767df2e4e05",
@ -47,24 +38,6 @@ version_sha256s = {
"daml_ledger": "5aab9a6cbdc987fc4279481152ff65bda503425f7e338c5123237b283aae44d6",
"daml_react": "30ab9db8a20df6cbfed7c1b42a45ca0ea55af6e150c013105d46c94afa5f9a46",
},
"1.0.1-snapshot.20200417.3908.1.722bac90": {
"linux": "aaf832ceda1a66a8469460d5a4b6c14f681ce692d4e9ef6010896febbaf4b6e1",
"macos": "c2f89e394332b6ff19f547ccb399bacd3cd50d2493249c2d3a1ecaad0b87ac8b",
"windows": "7033ef0c5cbe75910a27730643076d3919b5694f3dc75f8daa57d5c22d04c593",
"test_tool": "762cd4836a8359dca0fb3271ba2e1d0629138f7d8d914298324418a174c5d22a",
"daml_types": "5ef8c162c0ea7d9bb8e40e2ea1bc632e0e842c33d2da6b45ad9e14c34fe41d66",
"daml_ledger": "af92004ec98f9439785a3a8cf9659341ee000c47a0d63d0088668915ed1462e7",
"daml_react": "562609f975c4d98d202b8400cb1f80398c7b38195546b73cb57d3257ec384446",
},
"1.0.1-snapshot.20200424.3917.0.16093690": {
"linux": "c46d3b24b34078e974a958cd30deb8628a42d85ac2c0159341ad13eff25a7d22",
"macos": "f8cae6ff524d0050e323218196d47c7c8ecd0399dca8a0571324ca3f4380da23",
"windows": "2c884302b4cc80777cd084dc68d58f00102a043e4f1ac001d3be023d743e2670",
"test_tool": "a88bf4594aea82a218871b6966d298ddafcd7a9deb83e337e83ed100e86ea316",
"daml_types": "6ce2173fe407e6e5c9496b58a66f48db8f191c6c2bf66d1db3e2318d12d12f62",
"daml_ledger": "872733ef3dbd19a972e360935d747e3673ab1b972f713db08ce89f5e83c802a3",
"daml_react": "705d7ad62204a389de5cfaf0c0cc50e071aedc9168247ea806a64dc430f7c306",
},
"1.0.1": {
"linux": "9cff04c29bb28503b41dcde310a2f3307984b1d976f8bccfb38268672e730c8f",
"macos": "a7e094e2d8766c852a247e0601b7c062c435b0d91f9ab256bbf4fcb40971ee36",
@ -74,36 +47,6 @@ version_sha256s = {
"daml_ledger": "12fc3ef723171162128fb5951dec5452f75b1a3de7facf85a2b0126f46de159b",
"daml_react": "dc4cbf95f22cc0300af1f450be316bd55ebbc2816e9806231b13edce85bea44c",
},
"1.1.0-snapshot.20200422.3991.0.6391ee9f": {
"linux": "d3bddaa903ebaebb6f714a3ac39598ba7fd71e8b80636c1275054ed7b883a0d9",
"macos": "66f1713057800ed75db1b967a8ea2d9b6c18da1a76b8224abdec0d33ed5533ae",
"windows": "2c779776d923d990870fbb2aa1c71e6081258a9b9ccc7cf8242c419a765d4821",
"test_tool": "649ecf7a7e98caef7dac20082526444b33d85dfe79b4b9b66d069ad67aac74fa",
"daml_types": "cad498131eb60f2651303cc26d520fe3614750ef2705db23ffbefeae0be3b7cf",
"daml_ledger": "527b6353c6d88d79f69aa78a51b9707e541555c42f1a2331f2bf075a730b3305",
"daml_react": "18f3363aa9c4f32b1fac48028a73a56dfdd8a73058d4b8d119adb16c72cb70fb",
"create_daml_app_patch": "b187d446443209288c165cf34247307275b497e015a5d953805297c05279d856",
},
"1.1.0-snapshot.20200430.4057.0.681c862d": {
"linux": "199cb51a5b406c8968f2352fe04ace8368ce95beaee81f19073b0f998c5518bf",
"macos": "f4844d54a62999ced90a3435a6c7b97c21e96882151489451d88774fdd4a05bc",
"windows": "1367d103cd514c759dc55ff4600cc73dd68d67cf89fa6ec5260634b0bdad08d0",
"test_tool": "7f693a9384df622136d5a039fdd5ffd22a9fd12439064017f339002fbf509da2",
"daml_types": "3329d1c1fc7382e86af2bff305effd31e22387149dc3231e9ed03ccd1b25bc30",
"daml_ledger": "1c5a3eb196a90685f406d4a91a8514ab22d0ebeaf1302f22854c6ac1e49c5ae4",
"daml_react": "8a217c7af7c828f41b532640914ed83f3d19bf5b8b42a52671db427ed9dd770c",
"create_daml_app_patch": "b187d446443209288c165cf34247307275b497e015a5d953805297c05279d856",
},
"1.1.0-snapshot.20200506.4107.0.7e448d81": {
"linux": "36d192d7004b2d307437a97bf86af9ffdea8adde99419867cf51740dd7a89a94",
"macos": "2359fb405497784781c0c9a94737d6c1999e15a33789eda1372e7bea57068c71",
"windows": "d1e521ca254d6768acb4efc0f3e53e8f464a0f09948c60128fafdcddc96d2232",
"test_tool": "1f6dacd2e37022bb3d6de567408f4b0f009281115513e4c6a7c4ace11f093361",
"daml_types": "987560f3b2268bc8dffcbad2f257a182f7145884afc9c69c62658cea037459c1",
"daml_ledger": "3762cd5185d69121a3277fe2d3520cf2762e23e09049ef771d3e44c8c5891f2f",
"daml_react": "35d892da1844358e032adbfd987c5fcf7eb279209d846126c37c8d993401a792",
"create_daml_app_patch": "b187d446443209288c165cf34247307275b497e015a5d953805297c05279d856",
},
"1.1.1": {
"linux": "1e396287b7462147d182fd2c536a5ee03163e3efeefc989d368b1dfc01a40a97",
"macos": "f49dd3f5198d3392b81bd64067f697bc0d18612fe9599a50580d57d1dd9c721c",
@ -114,34 +57,34 @@ version_sha256s = {
"daml_react": "3dabcccd1d4c13637cc096c401606f777f90368a4057dc7220269f8101aa1888",
"create_daml_app_patch": "b187d446443209288c165cf34247307275b497e015a5d953805297c05279d856",
},
"1.2.0-snapshot.20200513.4172.0.021f4af3": {
"linux": "3cefc08a50cd2dc9809064e94fc34af93e9b95da211346483b8f32219d9d6b10",
"macos": "ee07b3c8ac414ee872afb598075957e26e7f1ff50a93e23a1df1a45844711a9a",
"windows": "a50b4c03d76eef967cb8061ed84d99b6175d3501f795b98d86c9a195d7b3e47c",
"test_tool": "9cdb47c4b3c2d49aba00a0e684beac105fdefa390aeecdd731c7e6811ab33105",
"daml_types": "5bc76ae50297af0587ec26d380bed46fd9ba90316000bdb7d0907c0db60171e5",
"daml_ledger": "6ca843d878a510910f4be34c61e0ad0b8f7a6d4646175334caecdda4f5a297d7",
"daml_react": "0fc2cd41c8f0205f07e18e066a2d7f89cbcb03ef46825f241e2ec1efa6cd0778",
"1.2.0": {
"linux": "3920012a013d2b162f1328addaaf6c8f322f0473c1f681146fa748039f838a77",
"macos": "2f0e39f864b288de7a8b55996916452c3226669054423ee921660eb3a3467b9f",
"windows": "0df31d5dda5a23ad48c0201135c0611881bdc49d7f5ecbf31fce24d2129e9626",
"test_tool": "7c41476ec2fff62b0b588f07cd7496b5fde8fcdb440a48d539041ac3df7141d6",
"daml_types": "339591a0213d33d55f73fc10f33edd8e7f3d1f6a7b560a0abddb5510e5fe9c26",
"daml_ledger": "873452dce84fab87b456b1da1d9d5bc2a74cf4c6f3c6872e409a1b4c3dd170f1",
"daml_react": "bb58c2c51befe75f7f50a4ccf90bd9ed2593cdfff6ac1c9a98e0a80f5dbf82f7",
"create_daml_app_patch": "b187d446443209288c165cf34247307275b497e015a5d953805297c05279d856",
},
"1.2.0-snapshot.20200520.4224.0.2af134ca": {
"linux": "611a14502dc9db690310b13b282dc869bb6b4926954715e990e797fe99747586",
"macos": "e5f23672acb0bc8347e27b79b573b4b12c5eebd575b7b5ed83b7ee47247cc717",
"windows": "6a87a34e76add64caab98422d16fb229a6f34aa3c9b460893148e0498ebe6792",
"test_tool": "672848dbee588c9086c5cd7320bfd03bdc95233a49a97d198f1b863cb8bf7100",
"daml_types": "96baeea080cee765bedab95245fec39f8570e6e5c390cd2c3907f2ecf1b0d6b3",
"daml_ledger": "5a3c85c50af261664dfc12f706b93c3fec4e73450811a4432abe438b362cdd4e",
"daml_react": "b03a91cb4d9302adde0bd941fb8b107462eb9aa5c9f1a87ca71e8243542d5af8",
"1.3.0-snapshot.20200603.4345.0.1386abc0": {
"linux": "354c90e1e912d048a8f54fad91b1846c08306e95a7fcab42f9729c59ffbe1048",
"macos": "110955c30d1021e4433ecbb335ca9b846ced54b4084979ac03cc87e16990026f",
"windows": "7a235e60188a5aeaa1a5f27dbf2f978058a529a3eb699c07e67a5edcb1a393f4",
"test_tool": "524925034f3e12da026a34e983891ee8b55fe74f96b309bce1d260bddcfd8926",
"daml_types": "55f2353c1db1bdb682adbf20735b0704dca90044943c28b4efb631907a490f0a",
"daml_ledger": "8e0afc7786eb425ee66d4f3d87b4f1f20bf012511509dca3b1ab5bfa3bf33d77",
"daml_react": "b1bf4a8dcccc617a0216120ea01817d1dffd564264fc1091a0f860c81c70df51",
"create_daml_app_patch": "b187d446443209288c165cf34247307275b497e015a5d953805297c05279d856",
},
"1.2.0-snapshot.20200520.4228.0.595f1e27": {
"linux": "8c8e423608a341ee98e40c2f7c7b129ae4674e93fe8b74bf476180444ac9bd6f",
"macos": "6c621bc1dd9fbffa073118c0a255f7c9f2cabc06e5f9c281bb07b216c866c13c",
"windows": "6aaebe7a12013391dc784d9cbcacb554fb998cf0c078ca18bb4059ff3358b87c",
"test_tool": "30b8d46b1e05000c169cd63c2e6002d129ca78e0648e0d2cb4c15ef6cafb406b",
"daml_types": "62c1a8bd81a751ce5c6b7fcd30b61fcf01c8fe27929172f0d939431f14ff371f",
"daml_ledger": "181b4a763c35da8e7241e36ace05f9b37fe48f88b570e645bbe2dab7cf6c0a63",
"daml_react": "029895aee31e4af3d995471af52466879163786659b20852bbb145d7a4063a13",
"1.3.0-snapshot.20200610.4412.0.0544323d": {
"linux": "50823f7a336ce53992bc64da54170e7e34a934095fa5276444b8ac34aec00251",
"macos": "2695c07b0b909fc557c622b80f743340155380a55bcdfe14f743cc7f9c147d86",
"windows": "19ab94b004f758d933fbc9c4a60caf6006b9b251714dd1de8a0c86dcd98e15c0",
"test_tool": "dd54d79ef4130fd397d0fc2eff5b89908cb161303f69d3b7c7f0aa152e06abb7",
"daml_types": "769d056a1e6474392f5b9ad4c931dfd82e8dea68a33155ae46711dc547b88607",
"daml_ledger": "db22d7a0ccde7e27628145060bd45c1a8caa4964454771912926e5e45d9961af",
"daml_react": "b4a892a076f92cec66e481e9f0a28997c31a7f969a3e1b8900e75295d4225345",
"create_daml_app_patch": "b187d446443209288c165cf34247307275b497e015a5d953805297c05279d856",
},
}

View File

@ -4,13 +4,17 @@
module Main (main) where
import Control.Concurrent.Async
import Control.Lens (view)
import Crypto.Hash (hashlazy, Digest, SHA256)
import Control.Lens ((.~), (&), (^?!), view, _Right)
import Control.Monad
import Crypto.Hash (digestFromByteString, hashlazy, Digest, SHA256)
import Data.Aeson
import Data.ByteArray.Encoding (Base(Base16), convertToBase)
import Data.ByteArray.Encoding (Base(Base16), convertFromBase, convertToBase)
import Data.ByteString (ByteString)
import qualified Data.ByteString.Lazy as BSL
import Data.Either (fromRight)
import Data.Either.Extra (eitherToMaybe)
import qualified Data.HashMap.Strict as HashMap
import Data.List
import Data.Map (Map)
import qualified Data.Map.Strict as Map
import Data.Set (Set)
@ -18,6 +22,7 @@ import qualified Data.Set as Set
import Data.SemVer (Version)
import qualified Data.SemVer as SemVer
import qualified Data.Text as T
import qualified Data.Text.Encoding as T
import Network.HTTP.Client (responseTimeout, responseTimeoutMicro)
import Network.HTTP.Simple
import Options.Applicative
@ -118,21 +123,29 @@ firstMessagingPatch =
getChecksums :: Version -> IO Checksums
getChecksums ver = do
putStrLn ("Requesting hashes for " <> SemVer.toString ver)
[ linuxHash, macosHash, windowsHash, testToolHash,
damlTypesHash, damlLedgerHash, damlReactHash] <-
req <- parseRequestThrow sha256Url
lines <- map T.words . T.lines . T.decodeUtf8 . BSL.toStrict . getResponseBody <$> httpLbs req
Just [linuxHash, macosHash, windowsHash] <- pure $
forM [sdkFilePath "linux", sdkFilePath "macos", sdkFilePath "windows"] $ \path -> do
-- This is fairly hacky but given that we only run this script
-- offline that seems fine for now.
(base16Hash : _) <- find (\line -> path == line !! 1) lines
byteHash <- (eitherToMaybe . convertFromBase Base16 . T.encodeUtf8) base16Hash
digestFromByteString @SHA256 @ByteString byteHash
[ testToolHash, damlTypesHash, damlLedgerHash, damlReactHash] <-
forConcurrently
[ sdkUrl "linux", sdkUrl "macos", sdkUrl "windows"
, testToolUrl
[ testToolUrl
, tsLib "types"
, tsLib "ledger"
, tsLib "react"
] getHash
mbCreateDamlAppPatchHash <- traverse getHash mbCreateDamlAppUrl
pure Checksums {..}
where sdkUrl platform =
where sdkFilePath platform = T.pack $
"./daml-sdk-" <> SemVer.toString ver <> "-" <> platform <> ".tar.gz"
sha256Url =
"https://github.com/digital-asset/daml/releases/download/v" <>
SemVer.toString ver <> "/" <>
"daml-sdk-" <> SemVer.toString ver <> "-" <> platform <> ".tar.gz"
SemVer.toString ver <> "/sha256sums"
testToolUrl =
"https://repo1.maven.org/maven2/com/daml/ledger-api-test-tool/" <>
SemVer.toString ver <> "/ledger-api-test-tool-" <> SemVer.toString ver <> ".jar"
@ -162,7 +175,16 @@ main = do
snapshotsReq <- parseRequestThrow "https://docs.daml.com/snapshots.json"
versionsResp <- httpJSON versionsReq
snapshotsResp <- httpJSON snapshotsReq
let allVersions = Versions (Set.filter (>= minimumVersion) (getVersions (getResponseBody versionsResp <> getResponseBody snapshotsResp)))
let stableVers = getVersions (getResponseBody versionsResp)
let allSnapshots = getVersions (getResponseBody snapshotsResp)
-- List of releases that we want to filter out snapshots for even
-- though they do not exist.
let skipped = Set.fromList [SemVer.fromText "1.1.0" ^?! _Right]
-- Only include snapshots for which there is no following stable version.
-- We do not simply filter for anything > than the latest stable version
-- since we might have a snapshot for a bugfix release.
let prunedSnapshots = Set.filter (\v -> toStable v `Set.notMember` (stableVers <> skipped)) allSnapshots
let allVersions = Versions (Set.filter (>= minimumVersion) (stableVers <> prunedSnapshots))
checksums <- mapM (\ver -> (ver,) <$> getChecksums ver) (Set.toList $ getVersions allVersions)
writeFileUTF8 outputFile (T.unpack $ renderVersionsFile allVersions $ Map.fromList checksums)
where toStable v = v & SemVer.release .~ [] & SemVer.metadata .~ []