Add a prototype for DAML Script dumps (#7934)

* Add a prototype for DAML Script dumps

This is still fairly rough unfortunately but it does at least have
some tests and it doesn’t interact with anything else, so hopefully we
can land this and then parallelize the work from there on.

changelog_begin
changelog_end

* Update daml-script/dump/src/main/scala/com/daml/script/dump/Encode.scala

Co-authored-by: Stefano Baghino <43749967+stefanobaghino-da@users.noreply.github.com>

* view all the things

changelog_begin
changelog_end

* Update daml-script/dump/src/main/scala/com/daml/script/dump/Dependencies.scala

Co-authored-by: Stefano Baghino <43749967+stefanobaghino-da@users.noreply.github.com>

* Fixup the switch to exists

changelog_begin
changelog_end

Co-authored-by: Stefano Baghino <43749967+stefanobaghino-da@users.noreply.github.com>
This commit is contained in:
Moritz Kiefer 2021-02-05 13:19:20 +01:00 committed by GitHub
parent e2c7dd05cc
commit cf3d0876af
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 1191 additions and 0 deletions

View File

@ -177,6 +177,7 @@ jobs:
//language-support/scala/... \
//ledger-api/... \
//ledger/... \
//daml-script/dump/... \
-//language-support/scala/examples/... \
-//language-support/scala/codegen-sample-app/... \
-//ledger/ledger-api-test-tool/... \
@ -188,6 +189,7 @@ jobs:
//language-support/scala/... \
//ledger-api/... \
//ledger/... \
//daml-script/dump/... \
-//libs-scala/gatling-utils/... \
-//language-support/scala/examples/... \
-//language-support/scala/codegen-sample-app/... \

View File

@ -0,0 +1,93 @@
# Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
load(
"//bazel_tools:scala.bzl",
"da_scala_binary",
"da_scala_library",
"da_scala_test",
)
da_scala_binary(
name = "dump",
srcs = glob(["src/main/scala/**/*.scala"]),
main_class = "com.daml.script.dump.Main",
resources = glob(["src/main/resources/**/*"]),
scala_deps = [
"@maven//:com_github_scopt_scopt",
"@maven//:com_typesafe_akka_akka_stream",
"@maven//:org_scalaz_scalaz_core",
"@maven//:org_typelevel_paiges_core",
],
visibility = ["//visibility:public"],
deps = [
"//daml-lf/archive:daml_lf_archive_reader",
"//daml-lf/archive:daml_lf_dev_archive_proto_java",
"//daml-lf/data",
"//daml-lf/language",
"//language-support/scala/bindings",
"//language-support/scala/bindings-akka",
"//ledger-api/rs-grpc-bridge",
"//ledger/ledger-api-client",
"//ledger/ledger-api-common",
"@maven//:org_apache_commons_commons_text",
],
)
da_scala_test(
name = "tests",
srcs = glob(["src/test/scala/**/*.scala"]),
scala_deps = [
"@maven//:org_scalatest_scalatest",
"@maven//:org_typelevel_paiges_core",
],
visibility = ["//visibility:public"],
deps = [
":dump",
"//daml-lf/data",
"//language-support/scala/bindings",
],
)
da_scala_test(
name = "integration-tests",
srcs = glob(["src/it/scala/**/*.scala"]),
data = [
"//compiler/damlc",
"//daml-script/daml:daml-script.dar",
"//ledger/test-common:dar-files",
],
resources = glob(["src/test/resources/**/*"]),
scala_deps = [
"@maven//:com_typesafe_akka_akka_actor",
"@maven//:com_typesafe_akka_akka_stream",
"@maven//:io_spray_spray_json",
"@maven//:org_scalatest_scalatest",
"@maven//:org_scalaz_scalaz_core",
],
visibility = ["//visibility:public"],
deps = [
":dump",
"//bazel_tools/runfiles:scala_runfiles",
"//daml-lf/archive:daml_lf_archive_reader",
"//daml-lf/archive:daml_lf_dev_archive_proto_java",
"//daml-lf/interpreter",
"//daml-lf/language",
"//daml-script/runner:script-runner-lib",
"//language-support/scala/bindings",
"//ledger-api/rs-grpc-bridge",
"//ledger-api/testing-utils",
"//ledger/ledger-api-auth",
"//ledger/ledger-api-client",
"//ledger/ledger-api-domain",
"//ledger/ledger-resources",
"//ledger/sandbox:sandbox-scala-tests-lib",
"//ledger/sandbox-common",
"//ledger/sandbox-common:sandbox-common-scala-tests-lib",
"//ledger/test-common",
"//libs-scala/ports",
"//libs-scala/resources",
"@maven//:io_grpc_grpc_api",
"@maven//:io_netty_netty_handler",
],
)

View File

@ -0,0 +1,251 @@
// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package com.daml.script.dump
import java.io.IOException
import java.nio.file.{Files, FileVisitResult, Path, SimpleFileVisitor}
import java.nio.file.attribute.BasicFileAttributes
import java.util.UUID
import akka.stream.scaladsl.Sink
import com.daml.bazeltools.BazelRunfiles
import com.daml.lf.language.Ast.Package
import com.daml.lf.data.Ref
import com.daml.lf.data.Ref.PackageId
import com.daml.lf.engine.script.{GrpcLedgerClient, Participants, Runner, ScriptTimeMode}
import com.daml.ledger.api.domain
import com.daml.ledger.api.refinements.ApiTypes.ApplicationId
import com.daml.ledger.api.testing.utils.{AkkaBeforeAndAfterAll, SuiteResourceManagementAroundEach}
import com.daml.ledger.api.v1.command_service.SubmitAndWaitRequest
import com.daml.ledger.api.v1.commands._
import com.daml.ledger.api.v1.ledger_offset.LedgerOffset
import com.daml.ledger.api.v1.transaction_filter.{Filters, TransactionFilter}
import com.daml.ledger.api.v1.{value => api}
import com.daml.ledger.client.LedgerClient
import com.daml.ledger.client.configuration.{
CommandClientConfiguration,
LedgerClientConfiguration,
LedgerIdRequirement,
}
import com.daml.lf.archive.{Dar, DarReader, Decode}
import com.daml.platform.sandbox.services.TestCommands
import com.daml.platform.sandboxnext.SandboxNextFixture
import scalaz.syntax.tag._
import scalaz.syntax.traverse._
import spray.json._
import scala.sys.process._
import org.scalatest.freespec.AsyncFreeSpec
import org.scalatest.matchers.should.Matchers
final class IT
extends AsyncFreeSpec
with Matchers
with AkkaBeforeAndAfterAll
with SuiteResourceManagementAroundEach
with SandboxNextFixture
with TestCommands {
private val appId = domain.ApplicationId("script-dump")
private val clientConfiguration = LedgerClientConfiguration(
applicationId = appId.unwrap,
ledgerIdRequirement = LedgerIdRequirement.none,
commandClient = CommandClientConfiguration.default,
sslContext = None,
token = None,
)
val isWindows: Boolean = sys.props("os.name").toLowerCase.contains("windows")
val exe = if (isWindows) { ".exe" }
else ""
private val tmpDir = Files.createTempDirectory("script_dump")
private val damlc =
BazelRunfiles.requiredResource(s"compiler/damlc/damlc$exe")
private val damlScriptLib = BazelRunfiles.requiredResource("daml-script/daml/daml-script.dar")
private def iouId(s: String) =
api.Identifier(packageId, moduleName = "Iou", s)
override protected def afterAll(): Unit = {
super.afterAll()
deleteRecursively(tmpDir)
}
// TODO(MK) Put this somewhere in //libs-scala
private def deleteRecursively(dir: Path): Unit = {
Files.walkFileTree(
dir,
new SimpleFileVisitor[Path] {
override def postVisitDirectory(dir: Path, exc: IOException) = {
Files.delete(dir)
FileVisitResult.CONTINUE
}
override def visitFile(file: Path, attrs: BasicFileAttributes) = {
Files.delete(file)
FileVisitResult.CONTINUE
}
},
)
()
}
private def submit(client: LedgerClient, p: Ref.Party, cmd: Command) =
client.commandServiceClient.submitAndWaitForTransaction(
SubmitAndWaitRequest(
Some(
Commands(
ledgerId = client.ledgerId.unwrap,
applicationId = appId.unwrap,
commandId = UUID.randomUUID().toString(),
party = p,
commands = Seq(cmd),
)
)
)
)
"Generated dump for IOU transfer compiles" in {
for {
client <- LedgerClient(channel, clientConfiguration)
p1 <- client.partyManagementClient.allocateParty(None, None).map(_.party)
p2 <- client.partyManagementClient.allocateParty(None, None).map(_.party)
t0 <- submit(
client,
p1,
Command().withCreate(
CreateCommand(
templateId = Some(iouId("Iou")),
createArguments = Some(
api.Record(
fields = Seq(
api.RecordField("issuer", Some(api.Value().withParty(p1))),
api.RecordField("owner", Some(api.Value().withParty(p1))),
api.RecordField("currency", Some(api.Value().withText("USD"))),
api.RecordField("amount", Some(api.Value().withNumeric("100"))),
api.RecordField("observers", Some(api.Value().withList(api.List()))),
)
)
),
)
),
)
cid0 = t0.getTransaction.events(0).getCreated.contractId
t1 <- submit(
client,
p1,
Command().withExercise(
ExerciseCommand(
templateId = Some(iouId("Iou")),
choice = "Iou_Split",
contractId = cid0,
choiceArgument = Some(
api
.Value()
.withRecord(
api.Record(fields =
Seq(api.RecordField(value = Some(api.Value().withNumeric("50"))))
)
)
),
)
),
)
cid1 = t1.getTransaction.events(1).getCreated.contractId
cid2 = t1.getTransaction.events(2).getCreated.contractId
t2 <- submit(
client,
p1,
Command().withExercise(
ExerciseCommand(
templateId = Some(iouId("Iou")),
choice = "Iou_Transfer",
contractId = cid2,
choiceArgument = Some(
api
.Value()
.withRecord(
api.Record(fields = Seq(api.RecordField(value = Some(api.Value().withParty(p2)))))
)
),
)
),
)
cid3 = t2.getTransaction.events(1).getCreated.contractId
_ <- submit(
client,
p2,
Command().withExercise(
ExerciseCommand(
templateId = Some(iouId("IouTransfer")),
choice = "IouTransfer_Accept",
contractId = cid3,
choiceArgument = Some(api.Value().withRecord(api.Record())),
)
),
)
_ <- Main.run(
Config(
ledgerHost = "localhost",
ledgerPort = serverPort.value,
parties = scala.List(p1, p2),
outputPath = tmpDir,
damlScriptLib = damlScriptLib.toString,
sdkVersion = "0.0.0",
)
)
_ = Seq[String](
damlc.toString,
"build",
"--project-root",
tmpDir.toString,
"-o",
tmpDir.resolve("dump.dar").toString,
).! shouldBe 0
// Now run the DAML Script again
p3 <- client.partyManagementClient.allocateParty(None, None).map(_.party)
p4 <- client.partyManagementClient.allocateParty(None, None).map(_.party)
encodedDar = DarReader().readArchiveFromFile(tmpDir.resolve("dump.dar").toFile).get
dar: Dar[(PackageId, Package)] = encodedDar
.map { case (pkgId, pkgArchive) => Decode.readArchivePayload(pkgId, pkgArchive) }
_ <- Runner.run(
dar,
Ref.Identifier(dar.main._1, Ref.QualifiedName.assertFromString("Dump:dump")),
inputValue = Some(JsArray(JsString(p3), JsString(p4))),
timeMode = ScriptTimeMode.WallClock,
initialClients = Participants(
default_participant = Some(new GrpcLedgerClient(client, ApplicationId("script"))),
participants = Map.empty,
party_participants = Map.empty,
),
)
transactions <- client.transactionClient
.getTransactions(
LedgerOffset().withBoundary(LedgerOffset.LedgerBoundary.LEDGER_BEGIN),
Some(LedgerOffset().withBoundary(LedgerOffset.LedgerBoundary.LEDGER_END)),
transactionFilter(p3),
)
.runWith(Sink.seq)
_ = transactions should have length 4
transactions <- client.transactionClient
.getTransactions(
LedgerOffset().withBoundary(LedgerOffset.LedgerBoundary.LEDGER_BEGIN),
Some(LedgerOffset().withBoundary(LedgerOffset.LedgerBoundary.LEDGER_END)),
transactionFilter(p4),
)
.runWith(Sink.seq)
_ = transactions should have length 2
acs <- client.activeContractSetClient
.getActiveContracts(transactionFilter(p3))
.runWith(Sink.seq)
_ = acs.flatMap(_.activeContracts) should have size 2
_ = println(acs)
acs <- client.activeContractSetClient
.getActiveContracts(transactionFilter(p4))
.runWith(Sink.seq)
_ = acs.flatMap(_.activeContracts) should have size 1
} yield succeed
}
private def transactionFilter(p: Ref.Party) =
TransactionFilter(filtersByParty = Seq(p -> Filters()).toMap)
}

View File

@ -0,0 +1,11 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<root level="INFO">
<appender-ref ref="STDOUT" />
</root>
</configuration>

View File

@ -0,0 +1,49 @@
// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package com.daml.script.dump
import java.nio.file.{Path}
import java.io.File
final case class Config(
ledgerHost: String,
ledgerPort: Int,
parties: List[String],
outputPath: Path,
sdkVersion: String,
damlScriptLib: String,
)
object Config {
def parse(args: Array[String]): Option[Config] =
parser.parse(args, Empty)
private val parser = new scopt.OptionParser[Config]("script-dump") {
opt[String]("host")
.required()
.action((x, c) => c.copy(ledgerHost = x))
opt[Int]("port")
.required()
.action((x, c) => c.copy(ledgerPort = x))
opt[String]("party")
.required()
.unbounded()
.action((x, c) => c.copy(parties = x :: c.parties))
opt[File]('o', "output")
.required()
.action((x, c) => c.copy(outputPath = x.toPath))
opt[String]("sdk-version")
.required()
.action((x, c) => c.copy(sdkVersion = x))
}
private val Empty = Config(
ledgerHost = "",
ledgerPort = -1,
parties = List(),
outputPath = null,
sdkVersion = "",
damlScriptLib = "daml-script",
)
}

View File

@ -0,0 +1,125 @@
// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package com.daml.script.dump
import java.io.{ByteArrayOutputStream, FileOutputStream}
import java.nio.file.Path
import java.util.jar.{Attributes, Manifest}
import java.util.zip.{ZipEntry, ZipOutputStream}
import com.daml.daml_lf_dev.DamlLf
import com.daml.ledger.client.LedgerClient
import com.daml.lf.archive.{Dar, Decode}
import com.daml.lf.archive.Reader.damlLfCodedInputStreamFromBytes
import com.daml.lf.data.Ref
import com.daml.lf.data.Ref.PackageId
import com.daml.lf.language.Ast
import com.google.protobuf.ByteString
import scala.annotation.tailrec
import scala.concurrent.{ExecutionContext, Future}
object Dependencies {
// Given a list of root package ids, download all packages transitively referenced by those roots.
def fetchPackages(client: LedgerClient, references: List[PackageId])(implicit
ec: ExecutionContext
): Future[Map[PackageId, (ByteString, Ast.Package)]] = {
def go(
todo: List[PackageId],
acc: Map[PackageId, (ByteString, Ast.Package)],
): Future[Map[PackageId, (ByteString, Ast.Package)]] =
todo match {
case Nil => Future.successful(acc)
case p :: todo if acc.contains(p) => go(todo, acc)
case p :: todo =>
client.packageClient.getPackage(p).flatMap { pkgResp =>
val cos = damlLfCodedInputStreamFromBytes(
pkgResp.archivePayload.toByteArray,
Decode.PROTOBUF_RECURSION_LIMIT,
)
val pkgId = PackageId.assertFromString(pkgResp.hash)
val pkg = Decode
.readArchivePayloadAndVersion(pkgId, DamlLf.ArchivePayload.parser().parseFrom(cos))
._1
._2
go(todo ++ pkg.directDeps, acc + (pkgId -> ((pkgResp.archivePayload, pkg))))
}
}
go(references, Map.empty)
}
def writeDar(
sdkVersion: String,
file: Path,
dar: Dar[(PackageId, ByteString, Ast.Package)],
): Unit = {
def encode(pkgId: PackageId, bs: ByteString) = {
DamlLf.Archive
.newBuilder()
.setHash(pkgId)
.setHashFunction(DamlLf.HashFunction.SHA256)
.setPayload(bs)
.build()
}
val out = new ZipOutputStream(new FileOutputStream(file.toFile))
out.putNextEntry(new ZipEntry("META-INF/MANIFEST.MF"))
out.write(manifest(sdkVersion, dar))
out.closeEntry()
dar.all.foreach { case (pkgId, bs, _) =>
out.putNextEntry(new ZipEntry(pkgId + ".dalf"))
out.write(encode(pkgId, bs).toByteArray)
out.closeEntry
}
out.close
}
private val providedLibraries: Set[Ref.PackageName] =
Set("daml-stdlib", "daml-prim", "daml-script").map(Ref.PackageName.assertFromString(_))
// Given the pkg id of a main dalf and the map of all downloaded packages produce
// a DAR or return None for builtin packages like daml-stdlib
// that dont need to be listed in data-dependencies.
def toDar(
pkgId: PackageId,
pkgs: Map[PackageId, (ByteString, Ast.Package)],
): Option[Dar[(PackageId, ByteString, Ast.Package)]] = {
def deps(pkgId: PackageId): Set[PackageId] = {
@tailrec
def go(todo: List[PackageId], acc: Set[PackageId]): Set[PackageId] =
todo match {
case Nil => acc
case p :: todo if acc.contains(p) => go(todo, acc)
case p :: todo =>
go(todo ++ pkgs(p)._2.directDeps.toList, acc.union(pkgs(p)._2.directDeps))
}
go(List(pkgId), Set.empty) - pkgId
}
for {
pkg <- pkgs.get(pkgId) if !pkg._2.metadata.exists(m => providedLibraries.contains(m.name))
} yield {
Dar(
(pkgId, pkg._1, pkg._2),
deps(pkgId).toList.map(pkgId => (pkgId, pkgs(pkgId)._1, pkgs(pkgId)._2)),
)
}
}
private def manifest[A, B](sdkVersion: String, dar: Dar[(PackageId, A, B)]): Array[Byte] = {
val manifest = new Manifest()
manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0")
manifest.getMainAttributes().put(new Attributes.Name("Format"), "daml-lf")
manifest
.getMainAttributes()
.put(new Attributes.Name("Dalfs"), dar.all.map(pkg => pkg._1 + ".dalf").mkString(", "))
manifest.getMainAttributes().put(new Attributes.Name("Main-Dalf"), dar.main._1 + ".dalf")
manifest.getMainAttributes().put(new Attributes.Name("Sdk-Version"), sdkVersion)
val bytes = new ByteArrayOutputStream()
manifest.write(bytes)
bytes.close
bytes.toByteArray
}
}

View File

@ -0,0 +1,253 @@
// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package com.daml.script.dump
import java.time.format.DateTimeFormatter
import java.time.{LocalDate, ZoneId, ZonedDateTime}
import com.daml.ledger.api.v1.transaction.{TransactionTree, TreeEvent}
import com.daml.ledger.api.v1.value.Value.Sum
import com.daml.ledger.api.v1.value.{Identifier, Record, RecordField, Value}
import com.daml.lf.data.Time.{Date, Timestamp}
import com.daml.script.dump.TreeUtils._
import org.apache.commons.text.StringEscapeUtils
import org.typelevel.paiges.Doc
import scalaz.std.list._
import scalaz.std.set._
import scalaz.syntax.foldable._
private[dump] object Encode {
def encodeTransactionTreeStream(trees: Seq[TransactionTree]): Doc = {
val parties = trees.toList.foldMap(partiesInTree(_))
val partyMap = partyMapping(parties)
val cids = trees.map(treeCids(_))
val cidMap = cidMapping(cids)
val refs = trees.toList.foldMap(treeRefs(_))
val moduleRefs = refs.map(_.moduleName).toSet
Doc.text("module Dump where") /
Doc.text("import Daml.Script") /
Doc.stack(moduleRefs.map(encodeImport(_))) /
Doc.hardLine +
encodePartyType(partyMap) /
Doc.hardLine +
encodeAllocateParties(partyMap) /
Doc.hardLine +
Doc.text("testDump : Script ()") /
(Doc.text("testDump = do") /
Doc.text("parties <- allocateParties") /
Doc.text("dump parties")).hang(2) /
Doc.hardLine +
Doc.text("dump : Parties -> Script ()") /
(Doc.text("dump Parties{..} = do") /
Doc.stack(trees.map(t => encodeTree(partyMap, cidMap, t))) /
Doc.text("pure ()")).hang(2)
}
private def encodeAllocateParties(partyMap: Map[String, String]): Doc =
Doc.text("allocateParties : Script Parties") /
(Doc.text("allocateParties = do") /
Doc.stack(partyMap.map { case (k, v) =>
Doc.text(v) + Doc.text(" <- allocateParty \"") + Doc.text(k) + Doc.text("\"")
}) /
Doc.text("pure Parties{..}")).hang(2)
private def encodePartyType(partyMap: Map[String, String]): Doc =
(Doc.text("data Parties = Parties with") /
Doc.stack(partyMap.values.map(p => Doc.text(p) + Doc.text(" : Party")))).hang(2)
private def encodeLocalDate(d: LocalDate): Doc = {
val formatter = DateTimeFormatter.ofPattern("uuuu MMM d")
Doc.text("(date ") + Doc.text(formatter.format(d)) + Doc.text(")")
}
private[dump] def encodeValue(
partyMap: Map[String, String],
cidMap: Map[String, String],
v: Value.Sum,
): Doc = {
def go(v: Value.Sum): Doc =
v match {
case Sum.Empty => throw new IllegalArgumentException("Empty value")
case Sum.Record(value) => encodeRecord(partyMap, cidMap, value)
// TODO Handle sums of products properly
case Sum.Variant(value) =>
parens(
qualifyId(value.getVariantId.copy(entityName = value.constructor)) +
Doc.text(" ") + go(value.getValue.sum)
)
case Sum.ContractId(c) => encodeCid(cidMap, c)
case Sum.List(value) =>
list(value.elements.map(v => go(v.sum)))
case Sum.Int64(i) => Doc.str(i)
case Sum.Numeric(i) => Doc.str(i)
case Sum.Text(t) =>
// Java-escaping rules should at least be reasonably close to Daml/Haskell.
Doc.text("\"") + Doc.text(StringEscapeUtils.escapeJava(t)) + Doc.text("\"")
case Sum.Party(p) => encodeParty(partyMap, p)
case Sum.Bool(b) =>
Doc.text(if (b) {
"True"
} else "False")
case Sum.Unit(_) => Doc.text("()")
case Sum.Timestamp(micros) =>
val t: ZonedDateTime = Timestamp.assertFromLong(micros).toInstant.atZone(ZoneId.of("UTC"))
val formatter = DateTimeFormatter.ofPattern("H m s")
parens(
Doc.text("time ") + encodeLocalDate(t.toLocalDate) + Doc.text(" ") + Doc.text(
formatter.format(t)
)
)
case Sum.Date(daysSinceEpoch) =>
val d = Date.assertFromDaysSinceEpoch(daysSinceEpoch)
encodeLocalDate(LocalDate.ofEpochDay(d.days.toLong))
case Sum.Optional(value) =>
value.value match {
case None => Doc.text("None")
case Some(v) => parens(Doc.text("Some ") + go(v.sum))
}
case Sum.Map(m) =>
parens(
Doc.text("TextMap.fromList ") +
list(m.entries.map(e => pair(go(Value.Sum.Text(e.key)), go(e.getValue.sum))))
)
case Sum.Enum(value) =>
qualifyId(value.getEnumId.copy(entityName = value.constructor))
case Sum.GenMap(m) =>
parens(
Doc.text("Map.fromList ") + list(
m.entries.map(e => pair(go(e.getKey.sum), go(e.getValue.sum)))
)
)
}
go(v)
}
private def parens(v: Doc) =
Doc.text("(") + v + Doc.text(")")
private def brackets(v: Doc) =
Doc.text("[") + v + Doc.text("]")
private def list(xs: Seq[Doc]) =
brackets(Doc.intercalate(Doc.text(", "), xs))
private def pair(v1: Doc, v2: Doc) =
parens(v1 + Doc.text(", ") + v2)
private def encodeRecord(
partyMap: Map[String, String],
cidMap: Map[String, String],
r: Record,
): Doc = {
if (r.fields.isEmpty) {
qualifyId(r.getRecordId)
} else {
(qualifyId(r.getRecordId) + Doc.text(" with") /
Doc.stack(r.fields.map(f => encodeField(partyMap, cidMap, f)))).nested(2)
}
}
private def encodeField(
partyMap: Map[String, String],
cidMap: Map[String, String],
field: RecordField,
): Doc =
Doc.text(field.label) + Doc.text(" = ") + encodeValue(partyMap, cidMap, field.getValue.sum)
private def encodeParty(partyMap: Map[String, String], s: String): Doc = Doc.text(partyMap(s))
private def encodeParties(partyMap: Map[String, String], ps: Iterable[String]): Doc =
Doc.text("[") +
Doc.intercalate(Doc.text(", "), ps.map(encodeParty(partyMap, _))) +
Doc.text("]")
private def encodeCid(cidMap: Map[String, String], cid: String): Doc = {
// LedgerStrings are strings that match the regexp ``[A-Za-z0-9#:\-_/ ]+
Doc.text(cidMap(cid))
}
private def qualifyId(id: Identifier): Doc =
Doc.text(id.moduleName) + Doc.text(".") + Doc.text(id.entityName)
private def encodeEv(
partyMap: Map[String, String],
cidMap: Map[String, String],
ev: TreeEvent.Kind,
): Doc = ev match {
case TreeEvent.Kind.Created(created) =>
Doc.text("createCmd ") + encodeRecord(partyMap, cidMap, created.getCreateArguments)
case TreeEvent.Kind.Exercised(exercised @ _) =>
Doc.text("exerciseCmd ") + encodeCid(cidMap, exercised.contractId) + Doc.space + encodeValue(
partyMap,
cidMap,
exercised.getChoiceArgument.sum,
)
case TreeEvent.Kind.Empty => throw new IllegalArgumentException("Unknown tree event")
}
private def bindCid(cidMap: Map[String, String], c: CreatedContract): Doc = {
Doc.text("let ") + encodeCid(cidMap, c.cid) + Doc.text(" = createdCid @") +
qualifyId(c.tplId) + Doc.text(" [") + Doc.intercalate(
Doc.text(", "),
c.path.map(encodeSelector(_)),
) + Doc.text("] tree")
}
private def encodeTree(
partyMap: Map[String, String],
cidMap: Map[String, String],
tree: TransactionTree,
): Doc = {
val rootEvs = tree.rootEventIds.map(tree.eventsById(_).kind)
val submitters = rootEvs.flatMap(evParties(_)).toSet
val cids = treeCids(tree)
(Doc.text("tree <- submitTreeMulti ") + encodeParties(partyMap, submitters) + Doc.text(
" [] do"
) /
Doc.stack(rootEvs.map(ev => encodeEv(partyMap, cidMap, ev)))).hang(2) /
Doc.stack(cids.map(bindCid(cidMap, _)))
}
private def encodeSelector(selector: Selector): Doc = Doc.str(selector.i)
private def encodeImport(moduleName: String) =
Doc.text("import qualified ") + Doc.text(moduleName)
private def partyMapping(parties: Set[String]): Map[String, String] = {
// - PartyIdStrings are strings that match the regexp ``[A-Za-z0-9:\-_ ]+``.
def safeParty(p: String) =
Seq(":", "-", "_", " ").foldLeft(p) { case (p, x) => p.replace(x, "") }.toLowerCase
// Map from original party id to Daml identifier
var partyMap: Map[String, String] = Map.empty
// Number of times weve gotten the same result from safeParty, we resolve collisions with a suffix.
var usedParties: Map[String, Int] = Map.empty
parties.foreach { p =>
val r = safeParty(p)
usedParties.get(r) match {
case None =>
partyMap += p -> s"${r}_0"
usedParties += r -> 0
case Some(value) =>
partyMap += p -> s"${r}_${value + 1}"
usedParties += r -> (value + 1)
}
}
partyMap
}
private def cidMapping(cids: Seq[Seq[CreatedContract]]): Map[String, String] = {
def lowerFirst(s: String) =
if (s.isEmpty) {
s
} else {
s.head.toLower.toString + s.tail
}
cids.view.zipWithIndex.flatMap { case (cs, treeIndex) =>
cs.view.zipWithIndex.map { case (c, i) =>
c.cid -> s"${lowerFirst(c.tplId.entityName)}_${treeIndex}_$i"
}
}.toMap
}
}

View File

@ -0,0 +1,129 @@
// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package com.daml.script.dump
import java.nio.charset.StandardCharsets
import java.nio.file.{Files, Path}
import akka.actor.ActorSystem
import akka.stream.Materializer
import akka.stream.scaladsl.Sink
import com.daml.grpc.adapter.{AkkaExecutionSequencerPool, ExecutionSequencerFactory}
import com.daml.ledger.api.v1.ledger_offset.LedgerOffset
import com.daml.ledger.api.v1.transaction.TransactionTree
import com.daml.ledger.api.v1.transaction_filter.{Filters, TransactionFilter}
import com.daml.ledger.client.LedgerClient
import com.daml.ledger.client.configuration.{
CommandClientConfiguration,
LedgerClientConfiguration,
LedgerIdRequirement,
}
import com.daml.lf.data.Ref.PackageId
import com.daml.lf.language.Ast
import com.google.protobuf.ByteString
import scalaz.std.list._
import scalaz.std.set._
import scalaz.syntax.foldable._
import scala.concurrent.duration.Duration
import scala.concurrent.{Await, ExecutionContext, Future}
import scala.util.control.NonFatal
object Main {
import TreeUtils._
def main(args: Array[String]): Unit = {
Config.parse(args) match {
case None => sys.exit(1)
case Some(config) => main(config)
}
}
def main(config: Config): Unit = {
implicit val sys: ActorSystem = ActorSystem("script-dump")
implicit val ec: ExecutionContext = sys.dispatcher
implicit val seq: ExecutionSequencerFactory = new AkkaExecutionSequencerPool("script-dump")
implicit val mat: Materializer = Materializer(sys)
run(config)
.recoverWith { case NonFatal(fail) =>
Future {
println(fail)
}
}
.onComplete(_ => sys.terminate())
Await.result(sys.whenTerminated, Duration.Inf)
()
}
def run(config: Config)(implicit
ec: ExecutionContext,
esf: ExecutionSequencerFactory,
mat: Materializer,
): Future[Unit] =
for {
client <- LedgerClient.singleHost(config.ledgerHost, config.ledgerPort, clientConfig)
trees <- client.transactionClient
.getTransactionTrees(
LedgerOffset(LedgerOffset.Value.Boundary(LedgerOffset.LedgerBoundary.LEDGER_BEGIN)),
Some(LedgerOffset(LedgerOffset.Value.Boundary(LedgerOffset.LedgerBoundary.LEDGER_END))),
filter(config.parties),
verbose = true,
)
.runWith(Sink.seq)
pkgRefs: Set[PackageId] = trees.toList
.foldMap(treeRefs(_))
.map(i => PackageId.assertFromString(i.packageId))
pkgs <- Dependencies.fetchPackages(client, pkgRefs.toList)
_ = writeDump(
config.sdkVersion,
config.damlScriptLib,
config.outputPath,
trees,
pkgRefs,
pkgs,
)
} yield ()
def writeDump(
sdkVersion: String,
damlScriptLib: String,
targetDir: Path,
trees: Seq[TransactionTree],
pkgRefs: Set[PackageId],
pkgs: Map[PackageId, (ByteString, Ast.Package)],
) = {
val dir = Files.createDirectories(targetDir)
Files.write(
dir.resolve("Dump.daml"),
Encode.encodeTransactionTreeStream(trees).render(80).getBytes(StandardCharsets.UTF_8),
)
val dars = pkgRefs.collect(Function.unlift(Dependencies.toDar(_, pkgs)))
val deps = Files.createDirectory(dir.resolve("deps"))
val depFiles = dars.zipWithIndex.map { case (dar, i) =>
val file = deps.resolve(dar.main._3.metadata.fold(i.toString)(_.name) + ".dar")
Dependencies.writeDar(sdkVersion, file, dar)
file
}
Files.write(
dir.resolve("daml.yaml"),
s"""sdk-version: $sdkVersion
|name: dump
|version: 1.0.0
|source: .
|dependencies: [daml-stdlib, daml-prim, $damlScriptLib]
|data-dependencies: [${depFiles.mkString(",")}]
|""".stripMargin.getBytes(StandardCharsets.UTF_8),
)
}
def filter(parties: List[String]): TransactionFilter =
TransactionFilter(parties.map(p => p -> Filters()).toMap)
val clientConfig: LedgerClientConfiguration = LedgerClientConfiguration(
applicationId = "script-dump",
ledgerIdRequirement = LedgerIdRequirement.none,
commandClient = CommandClientConfiguration.default,
sslContext = None,
)
}

View File

@ -0,0 +1,134 @@
// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package com.daml.script.dump
import com.daml.ledger.api.v1.transaction.{TransactionTree, TreeEvent}
import com.daml.ledger.api.v1.transaction.TreeEvent.Kind
import com.daml.ledger.api.v1.value.{Identifier, Value}
import com.daml.ledger.api.v1.value.Value.Sum
import scalaz.std.option._
import scalaz.std.list._
import scalaz.std.set._
import scalaz.syntax.foldable._
object TreeUtils {
final case class Selector(i: Int)
def traverseTree(tree: TransactionTree)(f: (List[Selector], TreeEvent.Kind) => Unit): Unit = {
def traverseEv(ev: TreeEvent.Kind, f: (List[Selector], TreeEvent.Kind) => Unit): Unit =
ev match {
case Kind.Empty =>
case created @ Kind.Created(_) =>
f(List(), created)
case exercised @ Kind.Exercised(value) =>
f(List(), exercised)
value.childEventIds.map(x => tree.eventsById(x).kind).zipWithIndex.foreach {
case (ev, i) =>
traverseEv(ev, { case (path, ev) => f(Selector(i) :: path, ev) })
}
}
tree.rootEventIds.map(tree.eventsById(_)).zipWithIndex.foreach { case (ev, i) =>
traverseEv(ev.kind, { case (path, ev) => f(Selector(i) :: path, ev) })
}
}
def partiesInTree(tree: TransactionTree): Set[String] = {
var parties: Set[String] = Set()
traverseTree(tree) { case (_, ev) =>
ev match {
case Kind.Empty =>
case Kind.Created(value) =>
parties = parties.union(valueParties(Value.Sum.Record(value.getCreateArguments)))
case Kind.Exercised(value) =>
parties = parties.union(valueParties(value.getChoiceArgument.sum))
}
}
parties
}
private def valueParties(v: Value.Sum): Set[String] = v match {
case Sum.Empty => Set()
case Sum.Record(value) =>
value.fields.map(v => valueParties(v.getValue.sum)).foldLeft(Set[String]()) { case (x, xs) =>
x.union(xs)
}
case Sum.Variant(value) => valueParties(value.getValue.sum)
case Sum.ContractId(_) => Set()
case Sum.List(value) =>
value.elements.map(v => valueParties(v.sum)).foldLeft(Set[String]()) { case (x, xs) =>
x.union(xs)
}
case Sum.Int64(_) => Set()
case Sum.Numeric(_) => Set()
case Sum.Text(_) => Set()
case Sum.Timestamp(_) => Set()
case Sum.Party(value) => Set(value)
case Sum.Bool(_) => Set()
case Sum.Unit(_) => Set()
case Sum.Date(_) => Set()
case Sum.Optional(value) => value.value.fold(Set[String]())(v => valueParties(v.sum))
case Sum.Map(value) =>
value.entries.map(e => valueParties(e.getValue.sum)).foldLeft(Set[String]()) { case (x, xs) =>
x.union(xs)
}
case Sum.Enum(_) => Set[String]()
case Sum.GenMap(value) =>
value.entries.map(e => valueParties(e.getValue.sum)).foldLeft(Set[String]()) { case (x, xs) =>
x.union(xs)
}
}
case class CreatedContract(cid: String, tplId: Identifier, path: List[Selector])
def treeCids(tree: TransactionTree): Seq[CreatedContract] = {
var cids: Seq[CreatedContract] = Seq()
traverseTree(tree) { case (selectors, kind) =>
kind match {
case Kind.Empty =>
case Kind.Exercised(_) =>
case Kind.Created(value) =>
cids ++= Seq(CreatedContract(value.contractId, value.getTemplateId, selectors))
}
}
cids
}
def evParties(ev: TreeEvent.Kind): Seq[String] = ev match {
case TreeEvent.Kind.Created(create) => create.signatories
case TreeEvent.Kind.Exercised(exercised) => exercised.actingParties
case TreeEvent.Kind.Empty => Seq()
}
def treeRefs(t: TransactionTree): Set[Identifier] =
t.eventsById.values.toList.foldMap(e => evRefs(e.kind))
def evRefs(e: TreeEvent.Kind): Set[Identifier] = e match {
case Kind.Empty => Set()
case Kind.Created(value) => valueRefs(Sum.Record(value.getCreateArguments))
case Kind.Exercised(value) => valueRefs(value.getChoiceArgument.sum)
}
def valueRefs(v: Value.Sum): Set[Identifier] = v match {
case Sum.Empty => Set()
case Sum.Record(value) =>
Set(value.getRecordId).union(value.fields.toList.foldMap(f => valueRefs(f.getValue.sum)))
case Sum.Variant(value) => Set(value.getVariantId).union(valueRefs(value.getValue.sum))
case Sum.ContractId(_) => Set()
case Sum.List(value) => value.elements.toList.foldMap(v => valueRefs(v.sum))
case Sum.Int64(_) => Set()
case Sum.Numeric(_) => Set()
case Sum.Text(_) => Set()
case Sum.Timestamp(_) => Set()
case Sum.Party(_) => Set()
case Sum.Bool(_) => Set()
case Sum.Unit(_) => Set()
case Sum.Date(_) => Set()
case Sum.Optional(value) => value.value.foldMap(v => valueRefs(v.sum))
case Sum.Map(value) => value.entries.toList.foldMap(e => valueRefs(e.getValue.sum))
case Sum.Enum(value) => Set(value.getEnumId)
case Sum.GenMap(value) =>
value.entries.toList.foldMap(e => valueRefs(e.getKey.sum).union(valueRefs(e.getValue.sum)))
}
}

View File

@ -0,0 +1,11 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<root level="INFO">
<appender-ref ref="STDOUT" />
</root>
</configuration>

View File

@ -0,0 +1,133 @@
// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package com.daml.script.dump
import com.daml.ledger.api.v1.{value => v}
import java.time.{Instant, LocalDate, OffsetDateTime, ZoneOffset}
import java.util.concurrent.TimeUnit
import com.google.protobuf.empty.Empty
import org.scalatest.freespec.AnyFreeSpec
import org.scalatest.matchers.should.Matchers
class EncodeValueSpec extends AnyFreeSpec with Matchers {
private def assertMicrosFromInstant(i: Instant): Long =
TimeUnit.SECONDS.toMicros(i.getEpochSecond) + TimeUnit.NANOSECONDS.toMicros(i.getNano.toLong)
import Encode._
"encodeValue" - {
"record" in {
val id1 = v.Identifier("pkg-id", "M", "R1")
val id2 = v.Identifier("pkg-id", "M", "R2")
val id3 = v.Identifier("pkg-id", "M", "R3")
val r = v.Value.Sum.Record(
v.Record(
Some(id1),
Seq(
v.RecordField("a", Some(v.Value().withInt64(1))),
v.RecordField(
"b",
Some(
v.Value()
.withRecord(
v.Record(Some(id2), Seq(v.RecordField("c", Some(v.Value().withInt64(42)))))
)
),
),
v.RecordField(
"c",
Some(v.Value().withRecord(v.Record(Some(id3)))),
),
),
)
)
encodeValue(Map.empty, Map.empty, r).render(80) shouldBe
"""M.R1 with
| a = 1
| b = M.R2 with
| c = 42
| c = M.R3""".stripMargin.replace("\r\n", "\n")
}
"variant" in {
val id = v.Identifier("pkg-id", "M", "V")
val variant =
v.Value().withVariant(v.Variant(Some(id), "Constr", Some(v.Value().withInt64(1))))
encodeValue(Map.empty, Map.empty, variant.sum).render(80) shouldBe "(M.Constr 1)"
}
"contract id" in {
val cid = v.Value().withContractId("my-contract-id")
encodeValue(Map.empty, Map("my-contract-id" -> "mapped_cid"), cid.sum)
.render(80) shouldBe "mapped_cid"
}
"list" in {
val l = v.Value().withList(v.List(Seq(v.Value().withInt64(0), v.Value().withInt64(1))))
encodeValue(Map.empty, Map.empty, l.sum).render(80) shouldBe "[0, 1]"
}
"int64" in {
encodeValue(Map.empty, Map.empty, v.Value().withInt64(42).sum).render(80) shouldBe "42"
}
"numeric" in {
encodeValue(Map.empty, Map.empty, v.Value().withNumeric("1.3000").sum)
.render(80) shouldBe "1.3000"
}
"text" in {
encodeValue(Map.empty, Map.empty, v.Value.Sum.Text("abc\"def"))
.render(80) shouldBe "\"abc\\\"def\""
}
"party" in {
encodeValue(Map("unmapped" -> "mapped"), Map.empty, v.Value.Sum.Party("unmapped"))
.render(80) shouldBe "mapped"
}
"bool" in {
encodeValue(Map.empty, Map.empty, v.Value.Sum.Bool(true)).render(80) shouldBe "True"
encodeValue(Map.empty, Map.empty, v.Value.Sum.Bool(false)).render(80) shouldBe "False"
}
"unit" in {
encodeValue(Map.empty, Map.empty, v.Value.Sum.Unit(Empty())).render(80) shouldBe "()"
}
"timestamp" in {
val date = OffsetDateTime.of(1999, 11, 16, 13, 37, 42, 0, ZoneOffset.UTC)
encodeValue(
Map.empty,
Map.empty,
v.Value.Sum.Timestamp(assertMicrosFromInstant(date.toInstant())),
).render(80) shouldBe "(time (date 1999 Nov 16) 13 37 42)"
}
"date" in {
val date = LocalDate.of(1999, 11, 16)
encodeValue(Map.empty, Map.empty, v.Value.Sum.Date(date.toEpochDay().toInt))
.render(80) shouldBe "(date 1999 Nov 16)"
}
"optional" in {
encodeValue(Map.empty, Map.empty, v.Value.Sum.Optional(v.Optional()))
.render(80) shouldBe "None"
encodeValue(
Map.empty,
Map.empty,
v.Value.Sum.Optional(v.Optional(Some(v.Value().withInt64(42)))),
).render(80) shouldBe "(Some 42)"
}
"textmap" in {
encodeValue(
Map.empty,
Map.empty,
v.Value.Sum.Map(v.Map(Seq(v.Map.Entry("key", Some(v.Value().withText("value")))))),
).render(80) shouldBe
"(TextMap.fromList [(\"key\", \"value\")])"
}
"enum" in {
val id = v.Identifier("pkg-id", "M", "E")
encodeValue(Map.empty, Map.empty, v.Value.Sum.Enum(v.Enum(Some(id), "Constr")))
.render(80) shouldBe "M.Constr"
}
"map" in {
val m = v.Value.Sum.GenMap(
v.GenMap(
Seq(v.GenMap.Entry(Some(v.Value().withInt64(42)), Some(v.Value().withText("value"))))
)
)
encodeValue(Map.empty, Map.empty, m).render(80) shouldBe "(Map.fromList [(42, \"value\")])"
}
}
}