From fdbc1f51ced2c6482402dc364b25ee5b0c177f3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Majcherski?= <43614029+majcherm-da@users.noreply.github.com> Date: Tue, 14 May 2019 16:18:55 +0200 Subject: [PATCH] windows: fixed daml-lf tests for Windows by using Bazel's rlocation (#1053) * windows: fixed daml-lf tests for Windows by using Bazel's rlocation * more consistent logging on CI; publishing Windows test logs on failure * windows: fix daml-lf engine tests * windows: add diff tool to msys --- bazel_tools/runfiles/BUILD | 10 +++++++ .../daml/bazeltools/BazelRunfiles.scala | 14 +++++++++ build.ps1 | 11 ++----- ci/build-unix.yml | 4 +-- ci/build-windows.yml | 10 +++++++ daml-lf/archive/BUILD.bazel | 1 + .../daml/lf/archive/DarReaderTest.scala | 5 ++-- .../archive/UniversalArchiveReaderSpec.scala | 8 +++-- daml-lf/engine/BUILD.bazel | 1 + .../daml/lf/engine/EngineTest.scala | 5 ++-- .../daml/lf/engine/LargeTransactionTest.scala | 5 ++-- daml-lf/testing-tools/BUILD.bazel | 1 + .../lf/engine/testing/SemanticTests.scala | 10 +++++-- daml-lf/tests/BUILD.bazel | 8 ++++- daml-lf/tests/daml-lf-test.sh | 28 +++++++++++++++-- daml-lf/tests/scenario/.gitignore | 1 + daml-lf/tests/scenario/test.sh | 30 ++++++++++++++++--- dev-env/windows/manifests/msys2.json | 2 +- 18 files changed, 124 insertions(+), 30 deletions(-) create mode 100644 bazel_tools/runfiles/BUILD create mode 100644 bazel_tools/runfiles/src/main/scala/com/digitalasset/daml/bazeltools/BazelRunfiles.scala diff --git a/bazel_tools/runfiles/BUILD b/bazel_tools/runfiles/BUILD new file mode 100644 index 0000000000..954f32dd63 --- /dev/null +++ b/bazel_tools/runfiles/BUILD @@ -0,0 +1,10 @@ +load("//bazel_tools:scala.bzl", "da_scala_library") + +da_scala_library( + name = "scala_runfiles", + srcs = glob(["src/main/scala/**/*.scala"]), + visibility = ["//visibility:public"], + deps = [ + "@bazel_tools//tools/java/runfiles", + ], +) diff --git a/bazel_tools/runfiles/src/main/scala/com/digitalasset/daml/bazeltools/BazelRunfiles.scala b/bazel_tools/runfiles/src/main/scala/com/digitalasset/daml/bazeltools/BazelRunfiles.scala new file mode 100644 index 0000000000..4e2cdb3ed4 --- /dev/null +++ b/bazel_tools/runfiles/src/main/scala/com/digitalasset/daml/bazeltools/BazelRunfiles.scala @@ -0,0 +1,14 @@ +// Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.digitalasset.daml.bazeltools + +import com.google.devtools.build.runfiles.Runfiles + +trait BazelRunfiles { + + private val MainWorkspace = "com_github_digital_asset_daml" + + def rlocation(path: String): String = Runfiles.create.rlocation(MainWorkspace + "/" + path) + +} diff --git a/build.ps1 b/build.ps1 index 6d8e36cb33..e32f165497 100644 --- a/build.ps1 +++ b/build.ps1 @@ -31,7 +31,7 @@ function bazel() { bazel shutdown # FIXME: Until all bazel issues on Windows are resolved we will be testing only specific bazel targets -bazel build `-`-experimental_execution_log_file ${ARTIFACT_DIRS}/build_full_execution_windows.log ` +bazel build `-`-experimental_execution_log_file ${ARTIFACT_DIRS}/build_execution_windows.log ` //release:sdk-release-tarball ` //release/windows-installer:windows-installer ` //:git-revision ` @@ -60,13 +60,8 @@ bazel run ` bazel shutdown -bazel test ` - //daml-lf/data/... ` - //daml-lf/interface/... ` - //daml-lf/interpreter/... ` - //daml-lf/lfpackage/... ` - //daml-lf/parser/... ` - //daml-lf/validation/... ` +bazel test `-`-experimental_execution_log_file ${ARTIFACT_DIRS}/test_execution_windows.log ` + //daml-lf/... ` //language-support/java/bindings/... ` //language-support/java/bindings-rxjava/... ` //ledger/ledger-api-client/... ` diff --git a/ci/build-unix.yml b/ci/build-unix.yml index b1009bfde5..c57a96b82f 100644 --- a/ci/build-unix.yml +++ b/ci/build-unix.yml @@ -36,14 +36,14 @@ steps: displayName: 'Publish the bazel execution logs' inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: 'Test logs' + artifactName: 'Execution logs' - task: PublishBuildArtifacts@1 condition: failed() displayName: 'Publish the bazel test logs' inputs: pathtoPublish: 'bazel-testlogs/' - artifactName: 'Bazel test logs' + artifactName: 'Test logs' - bash: ci/release.sh displayName: 'Release' diff --git a/ci/build-windows.yml b/ci/build-windows.yml index 4465b12e1b..576fd04a92 100644 --- a/ci/build-windows.yml +++ b/ci/build-windows.yml @@ -18,12 +18,20 @@ steps: pathtoPublish: '$(Build.ArtifactStagingDirectory)' artifactName: 'Execution logs' + - task: PublishBuildArtifacts@1 + condition: failed() + displayName: 'Publish the bazel test logs' + inputs: + pathtoPublish: 'bazel-testlogs/' + artifactName: 'Test logs' + - bash: | set -euo pipefail echo "Simulating release step..." echo "##vso[task.setvariable variable=has_released;isOutput=true]true" echo "##vso[task.setvariable variable=release_tag]$(cat VERSION)" name: release + - bash: | set -euo pipefail ARTIFACT=daml-sdk-$(release_tag)-windows.tar.gz @@ -34,11 +42,13 @@ steps: echo "##vso[task.setvariable variable=artifact-windows-installer;isOutput=true]$WINDOWS_INSTALLER" name: publish condition: eq(variables['release.has_released'], 'true') + - task: PublishPipelineArtifact@0 condition: eq(variables['release.has_released'], 'true') inputs: targetPath: $(Build.StagingDirectory)/$(publish.artifact) artifactName: $(publish.artifact) + - task: PublishPipelineArtifact@0 condition: eq(variables['release.has_released'], 'true') inputs: diff --git a/daml-lf/archive/BUILD.bazel b/daml-lf/archive/BUILD.bazel index 8086276264..7323c338de 100644 --- a/daml-lf/archive/BUILD.bazel +++ b/daml-lf/archive/BUILD.bazel @@ -121,6 +121,7 @@ da_scala_test_suite( "//3rdparty/jvm/org/scalatest", "//3rdparty/jvm/org/scalaz:scalaz_core", "//3rdparty/jvm/org/scalaz:scalaz_scalacheck_binding", + "//bazel_tools/runfiles:scala_runfiles", "//daml-lf/data", ], ) diff --git a/daml-lf/archive/src/test/scala/com/digitalasset/daml/lf/archive/DarReaderTest.scala b/daml-lf/archive/src/test/scala/com/digitalasset/daml/lf/archive/DarReaderTest.scala index 4f88da9a48..6782a77420 100644 --- a/daml-lf/archive/src/test/scala/com/digitalasset/daml/lf/archive/DarReaderTest.scala +++ b/daml-lf/archive/src/test/scala/com/digitalasset/daml/lf/archive/DarReaderTest.scala @@ -6,6 +6,7 @@ package com.digitalasset.daml.lf.archive import java.io.File import java.util.zip.ZipFile +import com.digitalasset.daml.bazeltools.BazelRunfiles import com.digitalasset.daml.lf.Dar import com.digitalasset.daml.lf.data.Ref import com.digitalasset.daml_lf.{DamlLf, DamlLf1} @@ -15,9 +16,9 @@ import scala.collection.JavaConverters._ import scala.util.{Success, Try} @SuppressWarnings(Array("org.wartremover.warts.Any")) -class DarReaderTest extends WordSpec with Matchers with Inside { +class DarReaderTest extends WordSpec with Matchers with Inside with BazelRunfiles { - private val darFile = resource("daml-lf/archive/DarReaderTest.dar") + private val darFile = resource(rlocation("daml-lf/archive/DarReaderTest.dar")) private def resource(path: String): File = { val f = new File(path).getAbsoluteFile diff --git a/daml-lf/archive/src/test/scala/com/digitalasset/daml/lf/archive/UniversalArchiveReaderSpec.scala b/daml-lf/archive/src/test/scala/com/digitalasset/daml/lf/archive/UniversalArchiveReaderSpec.scala index d084560580..69ff0d5628 100644 --- a/daml-lf/archive/src/test/scala/com/digitalasset/daml/lf/archive/UniversalArchiveReaderSpec.scala +++ b/daml-lf/archive/src/test/scala/com/digitalasset/daml/lf/archive/UniversalArchiveReaderSpec.scala @@ -3,17 +3,19 @@ package com.digitalasset.daml.lf +import com.digitalasset.daml.bazeltools.BazelRunfiles + import java.io.File import org.scalatest.{FlatSpec, Inside, Matchers} import scala.util.{Success, Try} -class UniversalArchiveReaderSpec extends FlatSpec with Matchers with Inside { +class UniversalArchiveReaderSpec extends FlatSpec with Matchers with Inside with BazelRunfiles { - private val darFile = new File("daml-lf/archive/DarReaderTest.dar") + private val darFile = new File(rlocation("daml-lf/archive/DarReaderTest.dar")) - private val dalfFile = new File("daml-lf/archive/DarReaderTest.dalf") + private val dalfFile = new File(rlocation("daml-lf/archive/DarReaderTest.dalf")) behavior of classOf[UniversalArchiveReader[_]].getSimpleName diff --git a/daml-lf/engine/BUILD.bazel b/daml-lf/engine/BUILD.bazel index a80d665973..f46dfef753 100644 --- a/daml-lf/engine/BUILD.bazel +++ b/daml-lf/engine/BUILD.bazel @@ -49,6 +49,7 @@ da_scala_test( "//3rdparty/jvm/org/scalacheck", "//3rdparty/jvm/org/scalatest", "//3rdparty/jvm/org/scalaz:scalaz_core", + "//bazel_tools/runfiles:scala_runfiles", "//daml-lf/archive:daml_lf_archive_scala", "//daml-lf/archive:daml_lf_java_proto", "//daml-lf/data", diff --git a/daml-lf/engine/src/test/scala/com/digitalasset/daml/lf/engine/EngineTest.scala b/daml-lf/engine/src/test/scala/com/digitalasset/daml/lf/engine/EngineTest.scala index 799c7a5090..69d7854990 100644 --- a/daml-lf/engine/src/test/scala/com/digitalasset/daml/lf/engine/EngineTest.scala +++ b/daml-lf/engine/src/test/scala/com/digitalasset/daml/lf/engine/EngineTest.scala @@ -6,6 +6,7 @@ package com.digitalasset.daml.lf.engine import java.util import java.io.File +import com.digitalasset.daml.bazeltools.BazelRunfiles import com.digitalasset.daml.lf.data.Ref._ import com.digitalasset.daml.lf.data.{FrontStack, ImmArray, Ref, Time} import com.digitalasset.daml.lf.lfpackage.Ast._ @@ -32,13 +33,13 @@ import scala.language.implicitConversions "org.wartremover.warts.Serializable", "org.wartremover.warts.Product" )) -class EngineTest extends WordSpec with Matchers { +class EngineTest extends WordSpec with Matchers with BazelRunfiles { import EngineTest._ private def loadPackage(resource: String): (PackageId, Package, Map[PackageId, Package]) = { val packages = - UniversalArchiveReader().readFile(new File(resource)).get + UniversalArchiveReader().readFile(new File(rlocation(resource))).get val packagesMap = Map(packages.all.map { case (pkgId, pkgArchive) => Decode.readArchivePayloadAndVersion(pkgId, pkgArchive)._1 }: _*) diff --git a/daml-lf/engine/src/test/scala/com/digitalasset/daml/lf/engine/LargeTransactionTest.scala b/daml-lf/engine/src/test/scala/com/digitalasset/daml/lf/engine/LargeTransactionTest.scala index df3d37c8b7..50a85475f9 100644 --- a/daml-lf/engine/src/test/scala/com/digitalasset/daml/lf/engine/LargeTransactionTest.scala +++ b/daml-lf/engine/src/test/scala/com/digitalasset/daml/lf/engine/LargeTransactionTest.scala @@ -5,6 +5,7 @@ package com.digitalasset.daml.lf.engine import java.io.File +import com.digitalasset.daml.bazeltools.BazelRunfiles import com.digitalasset.daml.lf.data.Ref._ import com.digitalasset.daml.lf.data.{FrontStack, ImmArray, Ref, Time} import com.digitalasset.daml.lf.lfpackage.{Ast, Decode} @@ -23,11 +24,11 @@ import org.scalatest.{Assertion, Matchers, WordSpec} import scala.language.implicitConversions @SuppressWarnings(Array("org.wartremover.warts.Any")) -class LargeTransactionTest extends WordSpec with Matchers { +class LargeTransactionTest extends WordSpec with Matchers with BazelRunfiles { private def loadPackage(resource: String): (PackageId, Package, Map[PackageId, Package]) = { val packages = - UniversalArchiveReader().readFile(new File(resource)).get + UniversalArchiveReader().readFile(new File(rlocation(resource))).get val packagesMap = Map(packages.all.map { case (pkgId, pkgArchive) => Decode.readArchivePayloadAndVersion(pkgId, pkgArchive)._1 }: _*) diff --git a/daml-lf/testing-tools/BUILD.bazel b/daml-lf/testing-tools/BUILD.bazel index d64c8d7026..275adc4d3b 100644 --- a/daml-lf/testing-tools/BUILD.bazel +++ b/daml-lf/testing-tools/BUILD.bazel @@ -43,6 +43,7 @@ da_scala_test( deps = [ ":testing-tools", "//3rdparty/jvm/org/scalaz:scalaz_core", + "//bazel_tools/runfiles:scala_runfiles", "//daml-lf/archive:daml_lf_archive_scala", "//daml-lf/archive:daml_lf_java_proto", "//daml-lf/data", diff --git a/daml-lf/testing-tools/src/test/scala/com/digitalasset/daml/lf/engine/testing/SemanticTests.scala b/daml-lf/testing-tools/src/test/scala/com/digitalasset/daml/lf/engine/testing/SemanticTests.scala index 350ad2ffbf..0d4d38e4ad 100644 --- a/daml-lf/testing-tools/src/test/scala/com/digitalasset/daml/lf/engine/testing/SemanticTests.scala +++ b/daml-lf/testing-tools/src/test/scala/com/digitalasset/daml/lf/engine/testing/SemanticTests.scala @@ -3,6 +3,7 @@ package com.digitalasset.daml.lf.engine.testing +import com.digitalasset.daml.bazeltools.BazelRunfiles import com.digitalasset.daml.lf.lfpackage.Ast.{DValue, Package} import com.digitalasset.daml.lf.lfpackage.Decode import org.scalatest.concurrent.{IntegrationPatience, ScalaFutures} @@ -13,7 +14,12 @@ import java.io.File import scala.concurrent.ExecutionContext.Implicits.global -class SemanticTests extends WordSpec with Matchers with ScalaFutures with IntegrationPatience { +class SemanticTests + extends WordSpec + with Matchers + with ScalaFutures + with IntegrationPatience + with BazelRunfiles { val loader = Thread.currentThread().getContextClassLoader() // these files are generated by bazel. Add new tests here. @@ -27,7 +33,7 @@ class SemanticTests extends WordSpec with Matchers with ScalaFutures with Integr val testPackages = darFiles.map(file => { val packages = - UniversalArchiveReader().readFile(new File(file)).get + UniversalArchiveReader().readFile(new File(rlocation(file))).get val packagesMap = Map(packages.all.map { case (pkgId, pkgArchive) => Decode.readArchivePayloadAndVersion(pkgId, pkgArchive)._1 }: _*) diff --git a/daml-lf/tests/BUILD.bazel b/daml-lf/tests/BUILD.bazel index 616b5ea293..16fc59942e 100644 --- a/daml-lf/tests/BUILD.bazel +++ b/daml-lf/tests/BUILD.bazel @@ -47,6 +47,9 @@ daml_compile( "//daml-lf/repl", ":%s.dar" % name, ], + deps = [ + "@bazel_tools//tools/bash/runfiles", + ], ) for name in TEST_FILES ] @@ -54,7 +57,7 @@ daml_compile( [ sh_test( name = "test-scenario-1-3-" + file.split("/")[2], - size = "small", + size = "medium", srcs = ["scenario/test.sh"], args = [ "$(location //daml-lf/repl:repl)", @@ -67,6 +70,9 @@ daml_compile( file, "%s/EXPECTED.ledger" % "/".join(file.split("/")[0:3]), ], + deps = [ + "@bazel_tools//tools/bash/runfiles", + ], ) for file in glob(["scenario/daml-1.3/*/Test.daml"]) ] diff --git a/daml-lf/tests/daml-lf-test.sh b/daml-lf/tests/daml-lf-test.sh index 47a60b7cb1..43f71e8b0f 100755 --- a/daml-lf/tests/daml-lf-test.sh +++ b/daml-lf/tests/daml-lf-test.sh @@ -4,9 +4,31 @@ set -eu -DAML_LF_REPL=$1 -DAMLC=$2 -MAIN=$3 +# --- begin runfiles.bash initialization --- +# Copy-pasted from Bazel's Bash runfiles library (tools/bash/runfiles/runfiles.bash). +if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then + if [[ -f "$0.runfiles_manifest" ]]; then + export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest" + elif [[ -f "$0.runfiles/MANIFEST" ]]; then + export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST" + elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then + export RUNFILES_DIR="$0.runfiles" + fi +fi +if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then + source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash" +elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then + source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \ + "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)" +else + echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash" + exit 1 +fi +# --- end runfiles.bash initialization --- + +DAML_LF_REPL=$(rlocation "$TEST_WORKSPACE/$1") +DAMLC=$(rlocation "$TEST_WORKSPACE/$2") +MAIN=$(rlocation "$TEST_WORKSPACE/$3") TMPDIR=$(mktemp -d) cleanup() { diff --git a/daml-lf/tests/scenario/.gitignore b/daml-lf/tests/scenario/.gitignore index 2f72b69a6f..9acf645fd3 100644 --- a/daml-lf/tests/scenario/.gitignore +++ b/daml-lf/tests/scenario/.gitignore @@ -1,2 +1,3 @@ ACTUAL.* *.dalf +*.dar diff --git a/daml-lf/tests/scenario/test.sh b/daml-lf/tests/scenario/test.sh index 17c380b43c..a9f7219397 100755 --- a/daml-lf/tests/scenario/test.sh +++ b/daml-lf/tests/scenario/test.sh @@ -10,9 +10,31 @@ set -eu export LC_ALL="en_US.UTF-8" -REPL=$1 -DAMLC=$2 -TESTMAIN=$3 +# --- begin runfiles.bash initialization --- +# Copy-pasted from Bazel's Bash runfiles library (tools/bash/runfiles/runfiles.bash). +if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then + if [[ -f "$0.runfiles_manifest" ]]; then + export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest" + elif [[ -f "$0.runfiles/MANIFEST" ]]; then + export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST" + elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then + export RUNFILES_DIR="$0.runfiles" + fi +fi +if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then + source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash" +elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then + source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \ + "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)" +else + echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash" + exit 1 +fi +# --- end runfiles.bash initialization --- + +REPL=$(rlocation "$TEST_WORKSPACE/$1") +DAMLC=$(rlocation "$TEST_WORKSPACE/$2") +TESTMAIN=$(rlocation "$TEST_WORKSPACE/$3") TESTDIR="$(dirname $TESTMAIN)" TESTDAR="$TESTDIR/Main.dar" @@ -24,4 +46,4 @@ $DAMLC package --debug --target $TARGET $TESTMAIN 'main' -o $TESTDAR $REPL test Test:run $TESTDAR | sed '1d' | sed -E "$REGEX_HIDE_HASHES" > ${TESTDIR}/ACTUAL.ledger -diff ${PWD}/${TESTDIR}/ACTUAL.ledger ${PWD}/${TESTDIR}/EXPECTED.ledger +diff --strip-trailing-cr ${TESTDIR}/ACTUAL.ledger ${TESTDIR}/EXPECTED.ledger diff --git a/dev-env/windows/manifests/msys2.json b/dev-env/windows/manifests/msys2.json index 49a029b1d7..3199dfac4c 100644 --- a/dev-env/windows/manifests/msys2.json +++ b/dev-env/windows/manifests/msys2.json @@ -23,7 +23,7 @@ "post_install": [ "try { bash.exe -lc 'pacman --version' } catch { }", "bash.exe -lc 'pacman -S --noconfirm unzip zip mingw-w64-x86_64-gcc'", - "try { bash.exe -lc 'pacman -S --noconfirm tar' } catch { }", + "try { bash.exe -lc 'pacman -S --noconfirm tar diffutils' } catch { }", "mkdir $dir\\bin", "cp $dir\\mingw64\\bin\\*.dll $dir\\bin" ]