Merge pull request #220 from digital-asset/leo-1123-dar-reader

DarReader fat dar support

daml-lf/archive/src/main/scala/com/digitalasset/daml/lf/archive/Dar.scala

@@ -0,0 +1,8 @@
+// Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package com.digitalasset.daml.lf
+
+case class Dar[A](main: A, dependencies: List[A]) {
+  def all: List[A] = main :: dependencies
+}

daml-lf/archive/src/main/scala/com/digitalasset/daml/lf/archive/DarManifestReader.scala

@@ -0,0 +1,45 @@
+// Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package com.digitalasset.daml.lf
+
+import java.io.InputStream
+import java.util.jar.{Attributes, Manifest}
+
+import scala.collection.breakOut
+import scala.util.{Failure, Success, Try}
+
+object DarManifestReader {
+
+  private val supportedFormat = "daml-lf"
+
+  def dalfNames(is: InputStream): Try[Dar[String]] = {
+    val manifest = new Manifest(is)
+    val attributes = value(manifest.getMainAttributes) _
+    for {
+      mainDalf <- attributes("Main-Dalf")
+      allDalfs <- attributes("Dalfs")
+      format <- attributes("Format")
+      _ <- checkFormat(format)
+    } yield Dar(mainDalf, dependencies(allDalfs, mainDalf))
+  }
+
+  private def dependencies(other: String, main: String): List[String] = {
+    val deps: List[String] = other.split(',').map(_.trim)(breakOut)
+    deps.filter(x => x != main)
+  }
+
+  private def value(attributes: Attributes)(key: String): Try[String] =
+    Option(attributes.getValue(key)) match {
+      case None => failure(s"Cannot find attribute: $key")
+      case Some(x) => Success(x.trim)
+    }
+
+  private def checkFormat(format: String): Try[Unit] =
+    if (format == supportedFormat) Success(())
+    else failure(s"Unsupported format: $format")
+
+  private def failure(msg: String) = Failure(DarManifestReaderException(msg))
+
+  case class DarManifestReaderException(msg: String) extends IllegalStateException(msg)
+}

daml-lf/archive/src/main/scala/com/digitalasset/daml/lf/archive/DarReader.scala

@@ -2,36 +2,59 @@
 // SPDX-License-Identifier: Apache-2.0
 
 package com.digitalasset.daml.lf.archive
-import java.io.InputStream
-import java.util
+
+import java.io.{BufferedInputStream, InputStream}
 import java.util.zip.{ZipEntry, ZipFile}
 
-import scala.collection.JavaConverters._
+import com.digitalasset.daml.lf.{Dar, DarManifestReader}
+import com.digitalasset.daml.lf.data.TryOps.sequence
+import com.digitalasset.daml.lf.data.Ref
+import com.digitalasset.daml_lf.DamlLf
+import com.digitalasset.daml.lf.data.TryOps.Bracket.bracket
+
 import scala.util.Try
 
-class DarReader[A](parse: InputStream => A) {
-  def readArchive(darFile: ZipFile): Try[List[A]] = {
-    import com.digitalasset.daml.lf.archive.TryOps.sequence
-    val collector = util.stream.Collectors.toList[Try[A]]
-    val list: util.List[Try[A]] = darFile.stream
-      .filter(isDalfEntry)
-      .map[Try[A]](e => parseEntry(darFile, e))
-      .collect(collector)
-    sequence(list.asScala.toList)
-  }
+class DarReader[A](
+    readDalfNamesFromManifest: InputStream => Try[Dar[String]],
+    parseDalf: InputStream => Try[A]) {
 
-  private def isDalfEntry(e: ZipEntry): Boolean = e.getName.endsWith(".dalf")
+  private val manifestEntry = new ZipEntry("META-INF/MANIFEST.MF")
 
-  private def parseEntry(f: ZipFile, e: ZipEntry): Try[A] = Try {
-    val is = f.getInputStream(e)
-    try {
-      parse(is)
-    } finally {
-      is.close()
-    }
-  }
+  def readArchive(darFile: ZipFile): Try[Dar[A]] =
+    for {
+      names <- parseDalfNamesFromManifest(darFile): Try[Dar[String]]
+      main <- parseOne(darFile)(names.main): Try[A]
+      deps <- parseAll(darFile)(names.dependencies): Try[List[A]]
+    } yield Dar(main, deps)
+
+  private def parseDalfNamesFromManifest(darFile: ZipFile): Try[Dar[String]] =
+    bracket(Try(darFile.getInputStream(manifestEntry)))(close)
+      .flatMap(is => readDalfNamesFromManifest(is))
+
+  private def parseAll(f: ZipFile)(names: List[String]): Try[List[A]] =
+    sequence(names.map(parseOne(f)))
+
+  private def parseOne(f: ZipFile)(s: String): Try[A] =
+    bracket(getZipEntryInputStream(f, s))(close).flatMap(parseDalf)
+
+  private def getZipEntryInputStream(f: ZipFile, name: String): Try[InputStream] =
+    for {
+      e <- Try(new ZipEntry(name))
+      is <- Try(new BufferedInputStream(f.getInputStream(e)))
+    } yield is
+
+  private def close(is: InputStream): Try[Unit] = Try(is.close())
 }
 
-object DarReader extends DarReader(Reader.decodeArchiveFromInputStream)
+object DarReader {
+  def apply(): DarReader[(Ref.PackageId, DamlLf.ArchivePayload)] =
+    new DarReader(DarManifestReader.dalfNames, a => Try(Reader.decodeArchiveFromInputStream(a)))
 
-object DarReaderWithVersion extends DarReader(Reader.readArchiveAndVersion)
+  def apply[A](parseDalf: InputStream => Try[A]): DarReader[A] =
+    new DarReader(DarManifestReader.dalfNames, parseDalf)
+}
+
+object DarReaderWithVersion
+    extends DarReader[((Ref.PackageId, DamlLf.ArchivePayload), LanguageMajorVersion)](
+      DarManifestReader.dalfNames,
+      a => Try(Reader.readArchiveAndVersion(a)))

daml-lf/archive/src/main/scala/com/digitalasset/daml/lf/archive/TryOps.scala

@@ -1,18 +0,0 @@
-// Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
-// SPDX-License-Identifier: Apache-2.0
-
-package com.digitalasset.daml.lf.archive
-import scala.util.{Success, Try}
-
-private[archive] object TryOps {
-  def sequence[A](list: List[Try[A]]): Try[List[A]] = {
-    val zero: Try[List[A]] = Success(List.empty[A])
-    list.foldRight(zero)((a, as) => map2(a, as)(_ :: _))
-  }
-
-  def map2[A, B, C](ta: Try[A], tb: Try[B])(f: (A, B) => C): Try[C] =
-    for {
-      a <- ta
-      b <- tb
-    } yield f(a, b)
-}

daml-lf/archive/src/test/scala/com/digitalasset/daml/lf/archive/DarManifestReaderTest.scala

@@ -0,0 +1,79 @@
+// Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package com.digitalasset.daml.lf
+import java.io.{ByteArrayInputStream, InputStream}
+import java.nio.charset.Charset
+
+import com.digitalasset.daml.lf.DarManifestReader.DarManifestReaderException
+import org.scalatest.{Inside, Matchers, WordSpec}
+
+import scala.util.{Failure, Success}
+
+class DarManifestReaderTest extends WordSpec with Matchers with Inside {
+
+  private val unicode = Charset.forName("UTF-8")
+
+  "should read dalf names from manifest, real scenario with Dalfs line split" in {
+    val manifest = """Manifest-Version: 1.0
+      |Created-By: Digital Asset packager (DAML-GHC)
+      |Main-Dalf: com.digitalasset.daml.lf.archive:DarReaderTest:0.1.dalf
+      |Dalfs: com.digitalasset.daml.lf.archive:DarReaderTest:0.1.dalf, daml-pri
+      | m.dalf
+      |Format: daml-lf
+      |Encryption: non-encrypted""".stripMargin
+
+    val inputStream: InputStream = new ByteArrayInputStream(manifest.getBytes(unicode))
+    val actual = DarManifestReader.dalfNames(inputStream)
+
+    actual shouldBe Success(
+      Dar("com.digitalasset.daml.lf.archive:DarReaderTest:0.1.dalf", List("daml-prim.dalf")))
+
+    inputStream.close()
+  }
+
+  "should read dalf names from manifest, Main-Dalf returned in the head" in {
+    val manifest = """Main-Dalf: A.dalf
+                     |Dalfs: B.dalf, C.dalf, A.dalf, E.dalf
+                     |Format: daml-lf
+                     |Encryption: non-encrypted""".stripMargin
+
+    val inputStream: InputStream = new ByteArrayInputStream(manifest.getBytes(unicode))
+    val actual = DarManifestReader.dalfNames(inputStream)
+
+    actual shouldBe Success(Dar("A.dalf", List("B.dalf", "C.dalf", "E.dalf")))
+
+    inputStream.close()
+  }
+
+  "should read dalf names from manifest, can handle one Dalf per manifest" in {
+    val manifest = """Main-Dalf: A.dalf
+                     |Dalfs: A.dalf
+                     |Format: daml-lf
+                     |Encryption: non-encrypted""".stripMargin
+
+    val inputStream: InputStream = new ByteArrayInputStream(manifest.getBytes(unicode))
+    val actual = DarManifestReader.dalfNames(inputStream)
+
+    actual shouldBe Success(Dar("A.dalf", List.empty))
+
+    inputStream.close()
+  }
+
+  "should return failure if Format is not daml-lf" in {
+    val manifest = """Main-Dalf: A.dalf
+                     |Dalfs: B.dalf, C.dalf, A.dalf, E.dalf
+                     |Format: anything-different-from-daml-lf
+                     |Encryption: non-encrypted""".stripMargin
+
+    val inputStream: InputStream = new ByteArrayInputStream(manifest.getBytes(unicode))
+    val actual = DarManifestReader.dalfNames(inputStream)
+
+    inside(actual) {
+      case Failure(DarManifestReaderException(msg)) =>
+        msg shouldBe "Unsupported format: anything-different-from-daml-lf"
+    }
+
+    inputStream.close()
+  }
+}

daml-lf/archive/src/test/scala/com/digitalasset/daml/lf/archive/DarReaderTest.scala

@@ -6,13 +6,13 @@ package com.digitalasset.daml.lf.archive
 import java.io.File
 import java.util.zip.ZipFile
 
-import com.digitalasset.daml_lf.DamlLf
-import com.digitalasset.daml_lf.DamlLf1
+import com.digitalasset.daml.lf.Dar
+import com.digitalasset.daml.lf.data.Ref
+import com.digitalasset.daml_lf.{DamlLf, DamlLf1}
 import org.scalatest.{Inside, Matchers, WordSpec}
 
 import scala.collection.JavaConverters._
 import scala.util.{Success, Try}
-import com.digitalasset.daml.lf.data.Ref.SimpleString
 
 @SuppressWarnings(Array("org.wartremover.warts.Any"))
 class DarReaderTest extends WordSpec with Matchers with Inside {
@@ -25,30 +25,42 @@ class DarReaderTest extends WordSpec with Matchers with Inside {
     f
   }
 
-  s"should read dar file: $darFile" in {
-    val archives: Try[List[((SimpleString, DamlLf.ArchivePayload), LanguageMajorVersion)]] =
+  s"should read dar file: $darFile, main archive: DarReaderTest returned first" in {
+    val archives: Try[Dar[((Ref.PackageId, DamlLf.ArchivePayload), LanguageMajorVersion)]] =
       DarReaderWithVersion.readArchive(new ZipFile(darFile))
 
     inside(archives) {
       case Success(
-          ((packageId1, archive1), LanguageMajorVersion.V1) ::
-            ((packageId2, archive2), LanguageMajorVersion.V1) :: Nil) =>
+          Dar(
+            ((packageId1, archive1), LanguageMajorVersion.V1),
+            ((packageId2, archive2), LanguageMajorVersion.V1) :: Nil)) =>
         packageId1.underlyingString shouldNot be('empty)
         packageId2.underlyingString shouldNot be('empty)
         archive1.getDamlLf1.getModulesCount should be > 0
         archive2.getDamlLf1.getModulesCount should be > 0
+
         val archive1Modules = archive1.getDamlLf1.getModulesList.asScala
-        val archive2Modules = archive2.getDamlLf1.getModulesList.asScala
-        inside(
-          archive1Modules
-            .find(m => name(m.getName) == "DarReaderTest")
-            .orElse(archive2Modules.find(m => name(m.getName) == "DarReaderTest"))) {
+        inside(archive1Modules.find(m => name(m.getName) == "DarReaderTest")) {
           case Some(module) =>
             val actualTypes: Set[String] =
-              module.getDataTypesList.asScala.toSet
-                .map((t: DamlLf1.DefDataType) => name(t.getName))
+              module.getDataTypesList.asScala.toSet.map((t: DamlLf1.DefDataType) => name(t.getName))
             actualTypes shouldBe Set("Transfer", "Call2", "CallablePayout", "PayOut")
         }
+
+        val archive2Modules = archive2.getDamlLf1.getModulesList.asScala
+        val archive2ModuleNames: Set[String] = archive2Modules.map(m => name(m.getName)).toSet
+        archive2ModuleNames shouldBe Set(
+          "GHC.Prim",
+          "GHC.Types",
+          "GHC.Enum",
+          "GHC.Show",
+          "GHC.Num",
+          "GHC.Classes",
+          "Control.Exception.Base",
+          "GHC.Tuple",
+          "GHC.Err",
+          "GHC.Base",
+          "LibraryModules")
     }
   }
 

daml-lf/data/src/main/scala/com/digitalasset/daml/lf/data/TryOps.scala

@@ -0,0 +1,68 @@
+// Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package com.digitalasset.daml.lf.data
+
+import scala.util.{Failure, Success, Try}
+
+private[lf] object TryOps {
+  def sequence[A](list: List[Try[A]]): Try[List[A]] = {
+    val zero: Try[List[A]] = Success(List.empty[A])
+    list.foldRight(zero)((a, as) => map2(a, as)(_ :: _))
+  }
+
+  def map2[A, B, C](ta: Try[A], tb: Try[B])(f: (A, B) => C): Try[C] =
+    for {
+      a <- ta
+      b <- tb
+    } yield f(a, b)
+
+  object Bracket {
+
+    /**
+      * The following description borrowed from https://hackage.haskell.org/package/exceptions-0.10.1/docs/Control-Monad-Catch.html#v:bracket
+      * {{{
+      * Generalized abstracted pattern of safe resource acquisition and release in the face of errors.
+      * The first action "acquires" some value, which is "released" by the second action at the end.
+      * The third action "uses" the value and its result is the result of the bracket.
+      *
+      * If an error is thrown during the use, the release still happens before the error is rethrown.
+      * }}}
+      *
+      * If cleanup fails, it's error returned. If you don't like this behavior, don't return a Failure from the cleanup.
+      * If both flatMap and cleanup fail, flatMap's Failure returned.
+      *
+      * Examples:
+      *
+      * {{{
+      *  private def parseDalfNamesFromManifest(darFile: ZipFile): Try[Dar[String]] =
+      *    bracket(Try(darFile.getInputStream(manifestEntry)))(close)
+      *      .flatMap(is => readDalfNamesFromManifest(is))
+      *
+      *  private def parseOne(f: ZipFile)(s: String): Try[A] =
+      *    bracket(getZipEntryInputStream(f, s))(close).flatMap(parseDalf)
+      *
+      *  private def close(is: InputStream): Try[Unit] = Try(is.close())
+      * }}}
+      *
+      * @see https://hackage.haskell.org/package/exceptions-0.10.1/docs/Control-Monad-Catch.html#v:bracket
+      * @param fa   acquired resource
+      * @param cleanup  cleanup function that releases the acquired resource
+      */
+    def bracket[A, B](fa: Try[A])(cleanup: A => Try[B]): Bracket[A, B] = new Bracket(fa, cleanup)
+
+    final class Bracket[A, B](fa: Try[A], cleanup: A => Try[B]) {
+      def flatMap[C](f: A => Try[C]): Try[C] = {
+        val fc = fa.flatMap(a => f(a))
+        val fb = fa.flatMap(a => cleanup(a))
+        (fc, fb) match {
+          case (Success(_), Success(_)) => fc
+          case (e @ Failure(_), _) => e
+          case (Success(_), Failure(e)) => Failure(e)
+        }
+      }
+
+      def map[C](f: A => C): Try[C] = flatMap(a => Try(f(a)))
+    }
+  }
+}

daml-lf/data/src/test/scala/com/digitalasset/daml/lf/data/TryOpsTest.scala

@@ -0,0 +1,86 @@
+// Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package com.digitalasset.daml.lf.data
+
+import org.scalatest.{Matchers, WordSpec}
+import org.scalatest.prop.GeneratorDrivenPropertyChecks
+import com.digitalasset.daml.lf.data.TryOps.Bracket.bracket
+
+import scala.util.{Failure, Success, Try}
+
+class TryOpsTest extends WordSpec with Matchers with GeneratorDrivenPropertyChecks {
+
+  "bracket should call clean after successful calculation" in forAll { (a: Int, b: Int) =>
+    var calls = List.empty[String]
+
+    def clean(x: Int): Try[Unit] = {
+      calls = s"clean $x" :: calls
+      Success(())
+    }
+
+    def add(x: Int)(y: Int): Try[Int] = {
+      calls = s"add $x $y" :: calls
+      Success(x + y)
+    }
+
+    val actual = bracket(Try(a))(clean).flatMap(add(b))
+    actual shouldBe Success(a + b)
+    calls.reverse shouldBe List(s"add $b $a", s"clean $a")
+  }
+
+  "bracket should fail if clean failed" in forAll { (a: Int, b: Int, e: Throwable) =>
+    var calls = List.empty[String]
+
+    def clean(x: Int): Try[Unit] = {
+      calls = s"clean $x $e" :: calls
+      Failure(e)
+    }
+
+    def add(x: Int)(y: Int): Try[Int] = {
+      calls = s"add $x $y" :: calls
+      Success(x + y)
+    }
+
+    val actual = bracket(Try(a))(clean).flatMap(add(b))
+    actual shouldBe Failure(e)
+    calls.reverse shouldBe List(s"add $b $a", s"clean $a $e")
+  }
+
+  "bracket should call clean if calculation fails" in forAll { (a: Int, b: Int, e: Throwable) =>
+    var calls = List.empty[String]
+
+    def clean(x: Int): Try[Unit] = {
+      calls = s"clean $x" :: calls
+      Success(())
+    }
+
+    def add(x: Int)(y: Int): Try[Int] = {
+      calls = s"add $x $y" :: calls
+      Failure(e)
+    }
+
+    val actual = bracket(Try(a))(clean).flatMap(add(b))
+    actual shouldBe Failure(e)
+    calls.reverse shouldBe List(s"add $b $a", s"clean $a")
+  }
+
+  "bracket should return calculation error if if both calculation and clean fail" in forAll {
+    (a: Int, b: Int, e1: Throwable, e2: Throwable) =>
+      var calls = List.empty[String]
+
+      def clean(x: Int): Try[Unit] = {
+        calls = s"clean $x $e2" :: calls
+        Failure(e2)
+      }
+
+      def add(x: Int)(y: Int): Try[Int] = {
+        calls = s"add $x $y" :: calls
+        Failure(e1)
+      }
+
+      val actual = bracket(Try(a))(clean).flatMap(add(b))
+      actual shouldBe Failure(e1)
+      calls.reverse shouldBe List(s"add $b $a", s"clean $a $e2")
+  }
+}

ledger/sandbox/src/main/scala/com/digitalasset/platform/sandbox/config/DamlPackageContainer.scala

@@ -14,6 +14,7 @@ import com.digitalasset.daml_lf.DamlLf.Archive
 import scala.collection.breakOut
 import scala.collection.immutable.Iterable
 import scala.util.control.NonFatal
+import scala.util.Try
 
 case class DamlPackageContainer(files: List[File] = Nil, devAllowed: Boolean = false) {
 
@@ -31,10 +32,9 @@ case class DamlPackageContainer(files: List[File] = Nil, devAllowed: Boolean = f
     }
 
   private def archivesFromDar(file: File): List[Archive] = {
-    new DarReader[Archive](Archive.parseFrom)
+    DarReader[Archive](x => Try(Archive.parseFrom(x)))
       .readArchive(new ZipFile(file))
-      .fold(t => throw new RuntimeException(s"Failed to parse DAR from $file", t), identity)
-
+      .fold(t => throw new RuntimeException(s"Failed to parse DAR from $file", t), dar => dar.all)
   }
 
   private def archivesFromDalf(file: File): List[Archive] = {