mirror of
https://github.com/digital-asset/daml.git
synced 2024-09-20 01:07:18 +03:00
Merge pull request #220 from digital-asset/leo-1123-dar-reader
DarReader fat dar support
This commit is contained in:
commit
2517fdcb2f
@ -0,0 +1,8 @@
|
||||
// Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package com.digitalasset.daml.lf
|
||||
|
||||
case class Dar[A](main: A, dependencies: List[A]) {
|
||||
def all: List[A] = main :: dependencies
|
||||
}
|
@ -0,0 +1,45 @@
|
||||
// Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package com.digitalasset.daml.lf
|
||||
|
||||
import java.io.InputStream
|
||||
import java.util.jar.{Attributes, Manifest}
|
||||
|
||||
import scala.collection.breakOut
|
||||
import scala.util.{Failure, Success, Try}
|
||||
|
||||
object DarManifestReader {
|
||||
|
||||
private val supportedFormat = "daml-lf"
|
||||
|
||||
def dalfNames(is: InputStream): Try[Dar[String]] = {
|
||||
val manifest = new Manifest(is)
|
||||
val attributes = value(manifest.getMainAttributes) _
|
||||
for {
|
||||
mainDalf <- attributes("Main-Dalf")
|
||||
allDalfs <- attributes("Dalfs")
|
||||
format <- attributes("Format")
|
||||
_ <- checkFormat(format)
|
||||
} yield Dar(mainDalf, dependencies(allDalfs, mainDalf))
|
||||
}
|
||||
|
||||
private def dependencies(other: String, main: String): List[String] = {
|
||||
val deps: List[String] = other.split(',').map(_.trim)(breakOut)
|
||||
deps.filter(x => x != main)
|
||||
}
|
||||
|
||||
private def value(attributes: Attributes)(key: String): Try[String] =
|
||||
Option(attributes.getValue(key)) match {
|
||||
case None => failure(s"Cannot find attribute: $key")
|
||||
case Some(x) => Success(x.trim)
|
||||
}
|
||||
|
||||
private def checkFormat(format: String): Try[Unit] =
|
||||
if (format == supportedFormat) Success(())
|
||||
else failure(s"Unsupported format: $format")
|
||||
|
||||
private def failure(msg: String) = Failure(DarManifestReaderException(msg))
|
||||
|
||||
case class DarManifestReaderException(msg: String) extends IllegalStateException(msg)
|
||||
}
|
@ -2,36 +2,59 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package com.digitalasset.daml.lf.archive
|
||||
import java.io.InputStream
|
||||
import java.util
|
||||
|
||||
import java.io.{BufferedInputStream, InputStream}
|
||||
import java.util.zip.{ZipEntry, ZipFile}
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
import com.digitalasset.daml.lf.{Dar, DarManifestReader}
|
||||
import com.digitalasset.daml.lf.data.TryOps.sequence
|
||||
import com.digitalasset.daml.lf.data.Ref
|
||||
import com.digitalasset.daml_lf.DamlLf
|
||||
import com.digitalasset.daml.lf.data.TryOps.Bracket.bracket
|
||||
|
||||
import scala.util.Try
|
||||
|
||||
class DarReader[A](parse: InputStream => A) {
|
||||
def readArchive(darFile: ZipFile): Try[List[A]] = {
|
||||
import com.digitalasset.daml.lf.archive.TryOps.sequence
|
||||
val collector = util.stream.Collectors.toList[Try[A]]
|
||||
val list: util.List[Try[A]] = darFile.stream
|
||||
.filter(isDalfEntry)
|
||||
.map[Try[A]](e => parseEntry(darFile, e))
|
||||
.collect(collector)
|
||||
sequence(list.asScala.toList)
|
||||
class DarReader[A](
|
||||
readDalfNamesFromManifest: InputStream => Try[Dar[String]],
|
||||
parseDalf: InputStream => Try[A]) {
|
||||
|
||||
private val manifestEntry = new ZipEntry("META-INF/MANIFEST.MF")
|
||||
|
||||
def readArchive(darFile: ZipFile): Try[Dar[A]] =
|
||||
for {
|
||||
names <- parseDalfNamesFromManifest(darFile): Try[Dar[String]]
|
||||
main <- parseOne(darFile)(names.main): Try[A]
|
||||
deps <- parseAll(darFile)(names.dependencies): Try[List[A]]
|
||||
} yield Dar(main, deps)
|
||||
|
||||
private def parseDalfNamesFromManifest(darFile: ZipFile): Try[Dar[String]] =
|
||||
bracket(Try(darFile.getInputStream(manifestEntry)))(close)
|
||||
.flatMap(is => readDalfNamesFromManifest(is))
|
||||
|
||||
private def parseAll(f: ZipFile)(names: List[String]): Try[List[A]] =
|
||||
sequence(names.map(parseOne(f)))
|
||||
|
||||
private def parseOne(f: ZipFile)(s: String): Try[A] =
|
||||
bracket(getZipEntryInputStream(f, s))(close).flatMap(parseDalf)
|
||||
|
||||
private def getZipEntryInputStream(f: ZipFile, name: String): Try[InputStream] =
|
||||
for {
|
||||
e <- Try(new ZipEntry(name))
|
||||
is <- Try(new BufferedInputStream(f.getInputStream(e)))
|
||||
} yield is
|
||||
|
||||
private def close(is: InputStream): Try[Unit] = Try(is.close())
|
||||
}
|
||||
|
||||
private def isDalfEntry(e: ZipEntry): Boolean = e.getName.endsWith(".dalf")
|
||||
object DarReader {
|
||||
def apply(): DarReader[(Ref.PackageId, DamlLf.ArchivePayload)] =
|
||||
new DarReader(DarManifestReader.dalfNames, a => Try(Reader.decodeArchiveFromInputStream(a)))
|
||||
|
||||
private def parseEntry(f: ZipFile, e: ZipEntry): Try[A] = Try {
|
||||
val is = f.getInputStream(e)
|
||||
try {
|
||||
parse(is)
|
||||
} finally {
|
||||
is.close()
|
||||
}
|
||||
}
|
||||
def apply[A](parseDalf: InputStream => Try[A]): DarReader[A] =
|
||||
new DarReader(DarManifestReader.dalfNames, parseDalf)
|
||||
}
|
||||
|
||||
object DarReader extends DarReader(Reader.decodeArchiveFromInputStream)
|
||||
|
||||
object DarReaderWithVersion extends DarReader(Reader.readArchiveAndVersion)
|
||||
object DarReaderWithVersion
|
||||
extends DarReader[((Ref.PackageId, DamlLf.ArchivePayload), LanguageMajorVersion)](
|
||||
DarManifestReader.dalfNames,
|
||||
a => Try(Reader.readArchiveAndVersion(a)))
|
||||
|
@ -1,18 +0,0 @@
|
||||
// Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package com.digitalasset.daml.lf.archive
|
||||
import scala.util.{Success, Try}
|
||||
|
||||
private[archive] object TryOps {
|
||||
def sequence[A](list: List[Try[A]]): Try[List[A]] = {
|
||||
val zero: Try[List[A]] = Success(List.empty[A])
|
||||
list.foldRight(zero)((a, as) => map2(a, as)(_ :: _))
|
||||
}
|
||||
|
||||
def map2[A, B, C](ta: Try[A], tb: Try[B])(f: (A, B) => C): Try[C] =
|
||||
for {
|
||||
a <- ta
|
||||
b <- tb
|
||||
} yield f(a, b)
|
||||
}
|
@ -0,0 +1,79 @@
|
||||
// Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package com.digitalasset.daml.lf
|
||||
import java.io.{ByteArrayInputStream, InputStream}
|
||||
import java.nio.charset.Charset
|
||||
|
||||
import com.digitalasset.daml.lf.DarManifestReader.DarManifestReaderException
|
||||
import org.scalatest.{Inside, Matchers, WordSpec}
|
||||
|
||||
import scala.util.{Failure, Success}
|
||||
|
||||
class DarManifestReaderTest extends WordSpec with Matchers with Inside {
|
||||
|
||||
private val unicode = Charset.forName("UTF-8")
|
||||
|
||||
"should read dalf names from manifest, real scenario with Dalfs line split" in {
|
||||
val manifest = """Manifest-Version: 1.0
|
||||
|Created-By: Digital Asset packager (DAML-GHC)
|
||||
|Main-Dalf: com.digitalasset.daml.lf.archive:DarReaderTest:0.1.dalf
|
||||
|Dalfs: com.digitalasset.daml.lf.archive:DarReaderTest:0.1.dalf, daml-pri
|
||||
| m.dalf
|
||||
|Format: daml-lf
|
||||
|Encryption: non-encrypted""".stripMargin
|
||||
|
||||
val inputStream: InputStream = new ByteArrayInputStream(manifest.getBytes(unicode))
|
||||
val actual = DarManifestReader.dalfNames(inputStream)
|
||||
|
||||
actual shouldBe Success(
|
||||
Dar("com.digitalasset.daml.lf.archive:DarReaderTest:0.1.dalf", List("daml-prim.dalf")))
|
||||
|
||||
inputStream.close()
|
||||
}
|
||||
|
||||
"should read dalf names from manifest, Main-Dalf returned in the head" in {
|
||||
val manifest = """Main-Dalf: A.dalf
|
||||
|Dalfs: B.dalf, C.dalf, A.dalf, E.dalf
|
||||
|Format: daml-lf
|
||||
|Encryption: non-encrypted""".stripMargin
|
||||
|
||||
val inputStream: InputStream = new ByteArrayInputStream(manifest.getBytes(unicode))
|
||||
val actual = DarManifestReader.dalfNames(inputStream)
|
||||
|
||||
actual shouldBe Success(Dar("A.dalf", List("B.dalf", "C.dalf", "E.dalf")))
|
||||
|
||||
inputStream.close()
|
||||
}
|
||||
|
||||
"should read dalf names from manifest, can handle one Dalf per manifest" in {
|
||||
val manifest = """Main-Dalf: A.dalf
|
||||
|Dalfs: A.dalf
|
||||
|Format: daml-lf
|
||||
|Encryption: non-encrypted""".stripMargin
|
||||
|
||||
val inputStream: InputStream = new ByteArrayInputStream(manifest.getBytes(unicode))
|
||||
val actual = DarManifestReader.dalfNames(inputStream)
|
||||
|
||||
actual shouldBe Success(Dar("A.dalf", List.empty))
|
||||
|
||||
inputStream.close()
|
||||
}
|
||||
|
||||
"should return failure if Format is not daml-lf" in {
|
||||
val manifest = """Main-Dalf: A.dalf
|
||||
|Dalfs: B.dalf, C.dalf, A.dalf, E.dalf
|
||||
|Format: anything-different-from-daml-lf
|
||||
|Encryption: non-encrypted""".stripMargin
|
||||
|
||||
val inputStream: InputStream = new ByteArrayInputStream(manifest.getBytes(unicode))
|
||||
val actual = DarManifestReader.dalfNames(inputStream)
|
||||
|
||||
inside(actual) {
|
||||
case Failure(DarManifestReaderException(msg)) =>
|
||||
msg shouldBe "Unsupported format: anything-different-from-daml-lf"
|
||||
}
|
||||
|
||||
inputStream.close()
|
||||
}
|
||||
}
|
@ -6,13 +6,13 @@ package com.digitalasset.daml.lf.archive
|
||||
import java.io.File
|
||||
import java.util.zip.ZipFile
|
||||
|
||||
import com.digitalasset.daml_lf.DamlLf
|
||||
import com.digitalasset.daml_lf.DamlLf1
|
||||
import com.digitalasset.daml.lf.Dar
|
||||
import com.digitalasset.daml.lf.data.Ref
|
||||
import com.digitalasset.daml_lf.{DamlLf, DamlLf1}
|
||||
import org.scalatest.{Inside, Matchers, WordSpec}
|
||||
|
||||
import scala.collection.JavaConverters._
|
||||
import scala.util.{Success, Try}
|
||||
import com.digitalasset.daml.lf.data.Ref.SimpleString
|
||||
|
||||
@SuppressWarnings(Array("org.wartremover.warts.Any"))
|
||||
class DarReaderTest extends WordSpec with Matchers with Inside {
|
||||
@ -25,30 +25,42 @@ class DarReaderTest extends WordSpec with Matchers with Inside {
|
||||
f
|
||||
}
|
||||
|
||||
s"should read dar file: $darFile" in {
|
||||
val archives: Try[List[((SimpleString, DamlLf.ArchivePayload), LanguageMajorVersion)]] =
|
||||
s"should read dar file: $darFile, main archive: DarReaderTest returned first" in {
|
||||
val archives: Try[Dar[((Ref.PackageId, DamlLf.ArchivePayload), LanguageMajorVersion)]] =
|
||||
DarReaderWithVersion.readArchive(new ZipFile(darFile))
|
||||
|
||||
inside(archives) {
|
||||
case Success(
|
||||
((packageId1, archive1), LanguageMajorVersion.V1) ::
|
||||
((packageId2, archive2), LanguageMajorVersion.V1) :: Nil) =>
|
||||
Dar(
|
||||
((packageId1, archive1), LanguageMajorVersion.V1),
|
||||
((packageId2, archive2), LanguageMajorVersion.V1) :: Nil)) =>
|
||||
packageId1.underlyingString shouldNot be('empty)
|
||||
packageId2.underlyingString shouldNot be('empty)
|
||||
archive1.getDamlLf1.getModulesCount should be > 0
|
||||
archive2.getDamlLf1.getModulesCount should be > 0
|
||||
|
||||
val archive1Modules = archive1.getDamlLf1.getModulesList.asScala
|
||||
val archive2Modules = archive2.getDamlLf1.getModulesList.asScala
|
||||
inside(
|
||||
archive1Modules
|
||||
.find(m => name(m.getName) == "DarReaderTest")
|
||||
.orElse(archive2Modules.find(m => name(m.getName) == "DarReaderTest"))) {
|
||||
inside(archive1Modules.find(m => name(m.getName) == "DarReaderTest")) {
|
||||
case Some(module) =>
|
||||
val actualTypes: Set[String] =
|
||||
module.getDataTypesList.asScala.toSet
|
||||
.map((t: DamlLf1.DefDataType) => name(t.getName))
|
||||
module.getDataTypesList.asScala.toSet.map((t: DamlLf1.DefDataType) => name(t.getName))
|
||||
actualTypes shouldBe Set("Transfer", "Call2", "CallablePayout", "PayOut")
|
||||
}
|
||||
|
||||
val archive2Modules = archive2.getDamlLf1.getModulesList.asScala
|
||||
val archive2ModuleNames: Set[String] = archive2Modules.map(m => name(m.getName)).toSet
|
||||
archive2ModuleNames shouldBe Set(
|
||||
"GHC.Prim",
|
||||
"GHC.Types",
|
||||
"GHC.Enum",
|
||||
"GHC.Show",
|
||||
"GHC.Num",
|
||||
"GHC.Classes",
|
||||
"Control.Exception.Base",
|
||||
"GHC.Tuple",
|
||||
"GHC.Err",
|
||||
"GHC.Base",
|
||||
"LibraryModules")
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -0,0 +1,68 @@
|
||||
// Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package com.digitalasset.daml.lf.data
|
||||
|
||||
import scala.util.{Failure, Success, Try}
|
||||
|
||||
private[lf] object TryOps {
|
||||
def sequence[A](list: List[Try[A]]): Try[List[A]] = {
|
||||
val zero: Try[List[A]] = Success(List.empty[A])
|
||||
list.foldRight(zero)((a, as) => map2(a, as)(_ :: _))
|
||||
}
|
||||
|
||||
def map2[A, B, C](ta: Try[A], tb: Try[B])(f: (A, B) => C): Try[C] =
|
||||
for {
|
||||
a <- ta
|
||||
b <- tb
|
||||
} yield f(a, b)
|
||||
|
||||
object Bracket {
|
||||
|
||||
/**
|
||||
* The following description borrowed from https://hackage.haskell.org/package/exceptions-0.10.1/docs/Control-Monad-Catch.html#v:bracket
|
||||
* {{{
|
||||
* Generalized abstracted pattern of safe resource acquisition and release in the face of errors.
|
||||
* The first action "acquires" some value, which is "released" by the second action at the end.
|
||||
* The third action "uses" the value and its result is the result of the bracket.
|
||||
*
|
||||
* If an error is thrown during the use, the release still happens before the error is rethrown.
|
||||
* }}}
|
||||
*
|
||||
* If cleanup fails, it's error returned. If you don't like this behavior, don't return a Failure from the cleanup.
|
||||
* If both flatMap and cleanup fail, flatMap's Failure returned.
|
||||
*
|
||||
* Examples:
|
||||
*
|
||||
* {{{
|
||||
* private def parseDalfNamesFromManifest(darFile: ZipFile): Try[Dar[String]] =
|
||||
* bracket(Try(darFile.getInputStream(manifestEntry)))(close)
|
||||
* .flatMap(is => readDalfNamesFromManifest(is))
|
||||
*
|
||||
* private def parseOne(f: ZipFile)(s: String): Try[A] =
|
||||
* bracket(getZipEntryInputStream(f, s))(close).flatMap(parseDalf)
|
||||
*
|
||||
* private def close(is: InputStream): Try[Unit] = Try(is.close())
|
||||
* }}}
|
||||
*
|
||||
* @see https://hackage.haskell.org/package/exceptions-0.10.1/docs/Control-Monad-Catch.html#v:bracket
|
||||
* @param fa acquired resource
|
||||
* @param cleanup cleanup function that releases the acquired resource
|
||||
*/
|
||||
def bracket[A, B](fa: Try[A])(cleanup: A => Try[B]): Bracket[A, B] = new Bracket(fa, cleanup)
|
||||
|
||||
final class Bracket[A, B](fa: Try[A], cleanup: A => Try[B]) {
|
||||
def flatMap[C](f: A => Try[C]): Try[C] = {
|
||||
val fc = fa.flatMap(a => f(a))
|
||||
val fb = fa.flatMap(a => cleanup(a))
|
||||
(fc, fb) match {
|
||||
case (Success(_), Success(_)) => fc
|
||||
case (e @ Failure(_), _) => e
|
||||
case (Success(_), Failure(e)) => Failure(e)
|
||||
}
|
||||
}
|
||||
|
||||
def map[C](f: A => C): Try[C] = flatMap(a => Try(f(a)))
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,86 @@
|
||||
// Copyright (c) 2019 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
package com.digitalasset.daml.lf.data
|
||||
|
||||
import org.scalatest.{Matchers, WordSpec}
|
||||
import org.scalatest.prop.GeneratorDrivenPropertyChecks
|
||||
import com.digitalasset.daml.lf.data.TryOps.Bracket.bracket
|
||||
|
||||
import scala.util.{Failure, Success, Try}
|
||||
|
||||
class TryOpsTest extends WordSpec with Matchers with GeneratorDrivenPropertyChecks {
|
||||
|
||||
"bracket should call clean after successful calculation" in forAll { (a: Int, b: Int) =>
|
||||
var calls = List.empty[String]
|
||||
|
||||
def clean(x: Int): Try[Unit] = {
|
||||
calls = s"clean $x" :: calls
|
||||
Success(())
|
||||
}
|
||||
|
||||
def add(x: Int)(y: Int): Try[Int] = {
|
||||
calls = s"add $x $y" :: calls
|
||||
Success(x + y)
|
||||
}
|
||||
|
||||
val actual = bracket(Try(a))(clean).flatMap(add(b))
|
||||
actual shouldBe Success(a + b)
|
||||
calls.reverse shouldBe List(s"add $b $a", s"clean $a")
|
||||
}
|
||||
|
||||
"bracket should fail if clean failed" in forAll { (a: Int, b: Int, e: Throwable) =>
|
||||
var calls = List.empty[String]
|
||||
|
||||
def clean(x: Int): Try[Unit] = {
|
||||
calls = s"clean $x $e" :: calls
|
||||
Failure(e)
|
||||
}
|
||||
|
||||
def add(x: Int)(y: Int): Try[Int] = {
|
||||
calls = s"add $x $y" :: calls
|
||||
Success(x + y)
|
||||
}
|
||||
|
||||
val actual = bracket(Try(a))(clean).flatMap(add(b))
|
||||
actual shouldBe Failure(e)
|
||||
calls.reverse shouldBe List(s"add $b $a", s"clean $a $e")
|
||||
}
|
||||
|
||||
"bracket should call clean if calculation fails" in forAll { (a: Int, b: Int, e: Throwable) =>
|
||||
var calls = List.empty[String]
|
||||
|
||||
def clean(x: Int): Try[Unit] = {
|
||||
calls = s"clean $x" :: calls
|
||||
Success(())
|
||||
}
|
||||
|
||||
def add(x: Int)(y: Int): Try[Int] = {
|
||||
calls = s"add $x $y" :: calls
|
||||
Failure(e)
|
||||
}
|
||||
|
||||
val actual = bracket(Try(a))(clean).flatMap(add(b))
|
||||
actual shouldBe Failure(e)
|
||||
calls.reverse shouldBe List(s"add $b $a", s"clean $a")
|
||||
}
|
||||
|
||||
"bracket should return calculation error if if both calculation and clean fail" in forAll {
|
||||
(a: Int, b: Int, e1: Throwable, e2: Throwable) =>
|
||||
var calls = List.empty[String]
|
||||
|
||||
def clean(x: Int): Try[Unit] = {
|
||||
calls = s"clean $x $e2" :: calls
|
||||
Failure(e2)
|
||||
}
|
||||
|
||||
def add(x: Int)(y: Int): Try[Int] = {
|
||||
calls = s"add $x $y" :: calls
|
||||
Failure(e1)
|
||||
}
|
||||
|
||||
val actual = bracket(Try(a))(clean).flatMap(add(b))
|
||||
actual shouldBe Failure(e1)
|
||||
calls.reverse shouldBe List(s"add $b $a", s"clean $a $e2")
|
||||
}
|
||||
}
|
@ -14,6 +14,7 @@ import com.digitalasset.daml_lf.DamlLf.Archive
|
||||
import scala.collection.breakOut
|
||||
import scala.collection.immutable.Iterable
|
||||
import scala.util.control.NonFatal
|
||||
import scala.util.Try
|
||||
|
||||
case class DamlPackageContainer(files: List[File] = Nil, devAllowed: Boolean = false) {
|
||||
|
||||
@ -31,10 +32,9 @@ case class DamlPackageContainer(files: List[File] = Nil, devAllowed: Boolean = f
|
||||
}
|
||||
|
||||
private def archivesFromDar(file: File): List[Archive] = {
|
||||
new DarReader[Archive](Archive.parseFrom)
|
||||
DarReader[Archive](x => Try(Archive.parseFrom(x)))
|
||||
.readArchive(new ZipFile(file))
|
||||
.fold(t => throw new RuntimeException(s"Failed to parse DAR from $file", t), identity)
|
||||
|
||||
.fold(t => throw new RuntimeException(s"Failed to parse DAR from $file", t), dar => dar.all)
|
||||
}
|
||||
|
||||
private def archivesFromDalf(file: File): List[Archive] = {
|
||||
|
Loading…
Reference in New Issue
Block a user