daml ledger export: export all parties (#10588)

* Add test-case to ConfigSpec for output type

changelog_begin
changelog_end

* Add an --all-parties flag to ledger export

changelog_begin
* [Daml export] You can now set the ``--all-parties`` option to generate
  a ledger export as seen by all known parties.
changelog_end

* Update docs

Co-authored-by: Andreas Herrmann <andreas.herrmann@tweag.io>

daml-script/export/integration-tests/export-lf-1.6/scala/com/daml/script/export/LF16ExportClient.scala

@@ -23,6 +23,7 @@ import com.daml.ledger.api.v1.value
 import com.daml.ledger.api.domain
 import com.daml.lf.archive.DarDecoder
 import com.daml.SdkVersion
+import com.daml.ledger.api.refinements.ApiTypes.Party
 
 import scala.concurrent.{Await, ExecutionContext, Future}
 import scala.concurrent.duration.Duration
@@ -196,7 +197,10 @@ object LF16ExportClient {
         Config.Empty.copy(
           ledgerHost = "localhost",
           ledgerPort = ledgerPort,
-          parties = Seq("Alice"),
+          partyConfig = PartyConfig(
+            parties = Seq.empty[Party],
+            allParties = true,
+          ),
           exportType = Some(
             Config.EmptyExportScript.copy(
               sdkVersion = SdkVersion.sdkVersion,

daml-script/export/integration-tests/matches-docs/scala/com/daml/script/export/ExampleExportClient.scala

@@ -9,6 +9,7 @@ import java.time.Duration
 
 import com.daml.SdkVersion
 import com.daml.fs.Utils.deleteRecursively
+import com.daml.ledger.api.refinements.ApiTypes.Party
 import com.daml.ledger.api.tls.TlsConfiguration
 import com.daml.lf.engine.script.{RunnerConfig, RunnerMain}
 
@@ -105,7 +106,10 @@ object ExampleExportClient {
         Config.Empty.copy(
           ledgerHost = "localhost",
           ledgerPort = clientConfig.targetPort,
-          parties = Seq("Alice", "Bob"),
+          partyConfig = PartyConfig(
+            allParties = false,
+            parties = Party.subst(Seq("Alice", "Bob")),
+          ),
           exportType = Some(
             Config.EmptyExportScript.copy(
               sdkVersion = SdkVersion.sdkVersion,

daml-script/export/integration-tests/reproduces-transactions/test/scala/com/daml/script/export/ReproducesTransactions.scala

@@ -13,7 +13,7 @@ import com.daml.lf.data.Ref
 import com.daml.lf.data.Ref.PackageId
 import com.daml.lf.engine.script.{Participants, Runner}
 import com.daml.ledger.api.domain
-import com.daml.ledger.api.refinements.ApiTypes.ApplicationId
+import com.daml.ledger.api.refinements.ApiTypes.{ApplicationId, Party}
 import com.daml.ledger.api.testing.utils.{AkkaBeforeAndAfterAll, SuiteResourceManagementAroundAll}
 import com.daml.ledger.api.v1.command_service.SubmitAndWaitRequest
 import com.daml.ledger.api.v1.commands._
@@ -129,7 +129,10 @@ trait ReproducesTransactions
         ledgerPort = serverPort.value,
         tlsConfig = TlsConfiguration(false, None, None, None),
         accessToken = None,
-        parties = parties,
+        partyConfig = PartyConfig(
+          parties = Party.subst(parties),
+          allParties = false,
+        ),
         start = offset,
         end = ledgerEnd,
         exportType = Some(

daml-script/export/src/main/scala/com/daml/script/export/Config.scala

@@ -7,6 +7,7 @@ import java.nio.file.{Path, Paths}
 import java.io.File
 
 import com.daml.auth.TokenHolder
+import com.daml.ledger.api.refinements.ApiTypes.Party
 import com.daml.ledger.api.tls.{TlsConfiguration, TlsConfigurationCli}
 import com.daml.ledger.api.v1.ledger_offset.LedgerOffset
 
@@ -15,7 +16,7 @@ final case class Config(
     ledgerPort: Int,
     tlsConfig: TlsConfiguration,
     accessToken: Option[TokenHolder],
-    parties: Seq[String],
+    partyConfig: PartyConfig,
     start: LedgerOffset,
     end: LedgerOffset,
     exportType: Option[ExportType],
@@ -30,6 +31,13 @@ final case class ExportScript(
     damlScriptLib: String,
 ) extends ExportType
 
+// This is a product rather than a sum, so that we can raise
+// an error of both --party and --all-parties is configured.
+final case class PartyConfig(
+    allParties: Boolean,
+    parties: Seq[Party],
+)
+
 object Config {
   def parse(args: Array[String]): Option[Config] =
     parser.parse(args, Empty)
@@ -63,12 +71,22 @@ object Config {
         "File from which the access token will be read, required to interact with an authenticated ledger."
       )
     opt[Seq[String]]("party")
-      .required()
       .unbounded()
-      .action((x, c) => c.copy(parties = c.parties ++ x.toList))
+      .action((x, c) =>
+        c.copy(partyConfig =
+          c.partyConfig.copy(parties = c.partyConfig.parties ++ Party.subst(x.toList))
+        )
+      )
       .text(
         "Export ledger state as seen by these parties. " +
-          "Pass --party multiple times or use a comma-separated list of party names to specify multiple parties."
+          "Pass --party multiple times or use a comma-separated list of party names to specify multiple parties. " +
+          "Use either --party or --all-parties but not both."
+      )
+    opt[Unit]("all-parties")
+      .action((_, c) => c.copy(partyConfig = c.partyConfig.copy(allParties = true)))
+      .text(
+        "Export ledger state as seen by all known parties. " +
+          "Use either --party or --all-parties but not both."
       )
     opt[String]("start")
       .optional()
@@ -115,6 +133,16 @@ object Config {
         case Some(_) => success
       }
     )
+    checkConfig(c => {
+      val pc = c.partyConfig
+      if (pc.allParties && pc.parties.nonEmpty) {
+        failure("Set either --party or --all-parties but not both at the same time")
+      } else if (!pc.allParties && pc.parties.isEmpty) {
+        failure("Set one of --party or --all-parties")
+      } else {
+        success
+      }
+    })
   }
 
   val EmptyExportScript = ExportScript(
@@ -140,7 +168,7 @@ object Config {
     ledgerPort = -1,
     tlsConfig = TlsConfiguration(false, None, None, None),
     accessToken = None,
-    parties = List(),
+    partyConfig = PartyConfig(parties = Seq.empty[Party], allParties = false),
     start = LedgerOffset(LedgerOffset.Value.Boundary(LedgerOffset.LedgerBoundary.LEDGER_BEGIN)),
     end = LedgerOffset(LedgerOffset.Value.Boundary(LedgerOffset.LedgerBoundary.LEDGER_END)),
     exportType = None,

daml-script/export/src/main/scala/com/daml/script/export/LedgerUtils.scala

@@ -5,7 +5,9 @@ package com.daml.script.export
 
 import akka.stream.Materializer
 import akka.stream.scaladsl.Sink
-import com.daml.ledger.api.refinements.ApiTypes.ContractId
+import com.daml.auth.TokenHolder
+import com.daml.ledger.api.domain
+import com.daml.ledger.api.refinements.ApiTypes.{ContractId, Party}
 import com.daml.ledger.api.v1.event.CreatedEvent
 import com.daml.ledger.api.v1.event.Event.Event
 import com.daml.ledger.api.v1.ledger_offset.LedgerOffset
@@ -13,10 +15,27 @@ import com.daml.ledger.api.v1.transaction.TransactionTree
 import com.daml.ledger.api.v1.transaction_filter.{Filters, TransactionFilter}
 import com.daml.ledger.client.LedgerClient
 
-import scala.concurrent.Future
+import scala.concurrent.{ExecutionContext, Future}
 
 object LedgerUtils {
 
+  /** Fetch all known parties from the ledger if requested by the given PartyConfig.
+    */
+  def getAllParties(
+      client: LedgerClient,
+      token: Option[TokenHolder],
+      config: PartyConfig,
+  )(implicit ec: ExecutionContext): Future[Seq[Party]] = {
+    if (config.allParties) {
+      val tokenString = token.flatMap(_.token)
+      def fromDetails(details: Seq[domain.PartyDetails]): Seq[Party] =
+        Party.subst(details.map(_.party))
+      client.partyManagementClient.listKnownParties(tokenString).map(fromDetails)
+    } else {
+      Future.successful(config.parties)
+    }
+  }
+
   /** Fetch the active contract set from the ledger.
     *
     * @param parties Fetch the ACS for these parties.
@@ -24,7 +43,7 @@ object LedgerUtils {
     */
   def getACS(
       client: LedgerClient,
-      parties: Seq[String],
+      parties: Seq[Party],
       offset: LedgerOffset,
   )(implicit
       mat: Materializer
@@ -36,7 +55,7 @@ object LedgerUtils {
       Future.successful(Map.empty)
     } else {
       client.transactionClient
-        .getTransactions(ledgerBegin, Some(offset), filter(parties), verbose = true)
+        .getTransactions(ledgerBegin, Some(offset), filter(Party.unsubst(parties)), verbose = true)
         .runFold(Map.empty[ContractId, CreatedEvent]) { case (acs, tx) =>
           tx.events.foldLeft(acs) { case (acs, ev) =>
             ev.event match {
@@ -57,7 +76,7 @@ object LedgerUtils {
     */
   def getTransactionTrees(
       client: LedgerClient,
-      parties: Seq[String],
+      parties: Seq[Party],
       start: LedgerOffset,
       end: LedgerOffset,
   )(implicit
@@ -67,7 +86,7 @@ object LedgerUtils {
       Future.successful(Seq.empty)
     } else {
       client.transactionClient
-        .getTransactionTrees(start, Some(end), filter(parties), verbose = true)
+        .getTransactionTrees(start, Some(end), filter(Party.unsubst(parties)), verbose = true)
         .runWith(Sink.seq)
     }
   }

daml-script/export/src/main/scala/com/daml/script/export/Main.scala

@@ -54,8 +54,9 @@ object Main {
             config.ledgerPort,
             clientConfig(config),
           )
-          acs <- LedgerUtils.getACS(client, config.parties, config.start)
-          trees <- LedgerUtils.getTransactionTrees(client, config.parties, config.start, config.end)
+          parties <- LedgerUtils.getAllParties(client, config.accessToken, config.partyConfig)
+          acs <- LedgerUtils.getACS(client, parties, config.start)
+          trees <- LedgerUtils.getTransactionTrees(client, parties, config.start, config.end)
           acsPkgRefs = TreeUtils.contractsReferences(acs.values)
           treePkgRefs = TreeUtils.treesReferences(trees)
           pkgRefs = acsPkgRefs ++ treePkgRefs

daml-script/export/src/test/scala/com/daml/script/export/ConfigSpec.scala

@@ -63,22 +63,41 @@ class ConfigSpec extends AnyFreeSpec with Matchers with OptionValues {
         optConfig.value.end shouldBe LedgerOffset().withAbsolute("00100")
       }
     }
-    "--party" - {
+    "--party or --all-parties" - {
       val defaultRequiredArgs = outputTypeArgs ++ outputArgs ++ sdkVersionArgs ++ ledgerArgs
       "--party Alice" in {
         val args = defaultRequiredArgs ++ Array("--party", "Alice")
         val optConfig = Config.parse(args)
-        optConfig.value.parties should contain only ("Alice")
+        optConfig.value.partyConfig.parties should contain only ("Alice")
+        optConfig.value.partyConfig.allParties shouldBe false
       }
       "--party Alice --party Bob" in {
         val args = defaultRequiredArgs ++ Array("--party", "Alice", "--party", "Bob")
         val optConfig = Config.parse(args)
-        optConfig.value.parties should contain only ("Alice", "Bob")
+        optConfig.value.partyConfig.parties should contain only ("Alice", "Bob")
+        optConfig.value.partyConfig.allParties shouldBe false
       }
       "--party Alice,Bob" in {
         val args = defaultRequiredArgs ++ Array("--party", "Alice,Bob")
         val optConfig = Config.parse(args)
-        optConfig.value.parties should contain only ("Alice", "Bob")
+        optConfig.value.partyConfig.parties should contain only ("Alice", "Bob")
+        optConfig.value.partyConfig.allParties shouldBe false
+      }
+      "--all-parties" in {
+        val args = defaultRequiredArgs ++ Array("--all-parties")
+        val optConfig = Config.parse(args)
+        optConfig.value.partyConfig.parties shouldBe empty
+        optConfig.value.partyConfig.allParties shouldBe true
+      }
+      "missing" in {
+        val args = defaultRequiredArgs
+        val optConfig = Config.parse(args)
+        optConfig shouldBe empty
+      }
+      "--party and --all-parties" in {
+        val args = defaultRequiredArgs ++ Array("--party", "Alice", "--all-parties")
+        val optConfig = Config.parse(args)
+        optConfig shouldBe empty
       }
     }
     "TLS" - {
@@ -118,5 +137,17 @@ class ConfigSpec extends AnyFreeSpec with Matchers with OptionValues {
         optConfig.value.accessToken.value.token.value shouldBe token
       }
     }
+    "Output type" - {
+      "missing" in {
+        val args = ledgerArgs ++ partyArgs
+        val optConfig = Config.parse(args)
+        optConfig shouldBe empty
+      }
+      "script" in {
+        val args = outputTypeArgs ++ outputArgs ++ sdkVersionArgs ++ ledgerArgs ++ partyArgs
+        val optConfig = Config.parse(args)
+        optConfig.value.exportType.value shouldBe an[ExportScript]
+      }
+    }
   }
 }

docs/source/tools/export/index.rst

@@ -39,8 +39,9 @@ with a running ledger, e.g. with a running ``daml start``.
 
 The ``--party`` flags define which contracts will be included in the export. In
 the above example only contracts visible to the parties Alice and Bob will be
-included in the export. Lack of visibility of certain events may cause
-references to :ref:`unknown contract ids <export-unknown-cids>`.
+included in the export. Alternatively, you can set ``--all-parties`` to export
+contracts seen by all known parties. Lack of visibility of certain events may
+cause references to :ref:`unknown contract ids <export-unknown-cids>`.
 
 The ``--output`` flag defines the directory prefix under which to generate the
 Daml project that contains the Daml script that represents the ledger export.