extractor, navigator: --ledger-api-inbound-message-size-max option replaces 50MiB hard-coded [package size] limit (#1520)

50MiB is no longer hard-coded on extractor, navigator input for sandbox
or any other server, permitting large packages; e.g. pass
--ledger-api-inbound-message-size-max 62914560 to extractor or navigator
to get a 60MiB limit.

Fixes #1463.

* extractor: --ledger-api-inbound-message-size-max option replaces 50MiB hard-coded [package size] limit

* navigator: make inbound buffer size configurable in navigator backend

* _actually_ change the max inbound size, scalafmt

* release note for --ledger-api-inbound-message-size-max option

* another release note for --ledger-api-inbound-message-size-max option

docs/source/support/release-notes.rst

@@ -12,6 +12,10 @@ HEAD — ongoing
 SQL Extractor
 ~~~~~~~~~~~~~
 
+- 50MiB is no longer hard-coded on extractor input for sandbox or any other server,
+  permitting large packages; e.g. pass ``--ledger-api-inbound-message-size-max 62914560``
+  to extractor to get a 60MiB limit.
+  See `#1520 <https://github.com/digital-asset/daml/pull/1520>`__.
 - Improving logging. See `#1518 <https://github.com/digital-asset/daml/pull/1518>`__.
 
 DAML Language
@@ -37,6 +41,10 @@ Navigator
 
 - Fixed a regression where Navigator console was not able to inspect contracts and events.
   See `#1454 <https://github.com/digital-asset/daml/issues/1454>`__.
+- 50MiB is no longer hard-coded on extractor input for sandbox or any other server,
+  permitting large packages; e.g. pass ``--ledger-api-inbound-message-size-max 62914560``
+  to extractor to get a 60MiB limit.
+  See `#1520 <https://github.com/digital-asset/daml/pull/1520>`__.
 
 
 Sandbox

extractor/src/main/scala/com/digitalasset/extractor/Extractor.scala

@@ -224,7 +224,7 @@ class Extractor[T <: Target](config: ExtractorConfig, target: T) extends StrictL
   private def createClient: Future[LedgerClient] = {
     val builder: NettyChannelBuilder = NettyChannelBuilder
       .forAddress(config.ledgerHost, config.ledgerPort)
-      .maxInboundMessageSize(50 * 1024 * 1024) // 50 MiBytes
+      .maxInboundMessageSize(config.ledgerInboundMessageSizeMax)
 
     config.tlsConfig.client
       .fold {

extractor/src/main/scala/com/digitalasset/extractor/config/ConfigParser.scala

@@ -40,6 +40,7 @@ object ConfigParser {
       postgresStripPrefix: Option[String] = None,
       ledgerHost: String = "127.0.0.1",
       ledgerPort: Int = 6865,
+      ledgerInboundMessageSizeMax: Int = 50 * 1024 * 1024, // 50 MiBytes
       party: ExtractorConfig.Parties = OneAnd(Party assertFromString "placeholder", Nil),
       templateConfigs: Set[TemplateConfig] = Set.empty,
       from: Option[String] = None,
@@ -173,6 +174,12 @@ object ConfigParser {
         .valueName("<p>")
         .text("The port of the Ledger host. Default is 6865.")
 
+      opt[Int]("ledger-api-inbound-message-size-max").optional
+        .validate(x => Either.cond(x > 0, (), "Message size must be positive"))
+        .action((x, c) => c.copy(ledgerInboundMessageSizeMax = x))
+        .valueName("<bytes>")
+        .text("Maximum message size from the ledger API. Default is 52428800 (50MiB).")
+
       opt[ExtractorConfig.Parties]("party")
         .required()
         .action((x, c) => c.copy(party = x))
@@ -290,7 +297,8 @@ object ConfigParser {
 
       val config = ExtractorConfig(
         cliParams.ledgerHost,
-        cliParams.ledgerPort,
+        ledgerPort = cliParams.ledgerPort,
+        ledgerInboundMessageSizeMax = cliParams.ledgerInboundMessageSizeMax,
         from,
         to,
         cliParams.party,

extractor/src/main/scala/com/digitalasset/extractor/config/ExtractorConfig.scala

@@ -24,6 +24,7 @@ object SnapshotEndSetting {
 final case class ExtractorConfig(
     ledgerHost: String,
     ledgerPort: Int,
+    ledgerInboundMessageSizeMax: Int,
     from: LedgerOffset,
     to: SnapshotEndSetting,
     parties: ExtractorConfig.Parties,

extractor/src/test/lib/scala/com/digitalasset/extractor/services/ExtractorFixture.scala

@@ -27,7 +27,8 @@ trait ExtractorFixture extends SandboxFixture with PostgresAround with Types {
 
   protected val baseConfig = ExtractorConfig(
     "127.0.0.1",
-    666, // doesn't matter, will/must be overriden in the test cases
+    ledgerPort = 666, // doesn't matter, will/must be overridden in the test cases
+    ledgerInboundMessageSizeMax = 50 * 1024 * 1024,
     LedgerOffset(LedgerOffset.Value.Boundary(LedgerOffset.LedgerBoundary.LEDGER_BEGIN)),
     SnapshotEndSetting.Head,
     OneAnd(Party assertFromString "Bob", List.empty),

navigator/backend/src/main/scala/com/digitalasset/navigator/UIBackend.scala

@@ -238,7 +238,8 @@ abstract class UIBackend extends LazyLogging with ApplicationInfoJsonSupport {
         arguments.platformPort,
         arguments.tlsConfig,
         arguments.time,
-        applicationInfo))
+        applicationInfo,
+        arguments.ledgerInboundMessageSizeMax))
     config.parties.foreach(store ! Subscribe(_))
 
     def graphQL: GraphQLHandler = DefaultGraphQLHandler(customEndpoints, Some(store))

navigator/backend/src/main/scala/com/digitalasset/navigator/config/Arguments.scala

@@ -29,7 +29,8 @@ case class Arguments(
     configFile: Option[Path] = None,
     startConsole: Boolean = false,
     startWebServer: Boolean = false,
-    useDatabase: Boolean = false
+    useDatabase: Boolean = false,
+    ledgerInboundMessageSizeMax: Int = 50 * 1024 * 1024 // 50 MiB
 )
 
 trait ArgumentsHelper { self: OptionParser[Arguments] =>
@@ -125,6 +126,16 @@ object Arguments {
               useDatabase = true
           ))
 
+      opt[Int]("ledger-api-inbound-message-size-max")
+        .hidden()
+        .text("Maximum message size from the ledger API. Default is 52428800 (50MiB).")
+        .valueName("<bytes>")
+        .validate(x => Either.cond(x > 0, (), "Buffer size must be positive"))
+        .action((x, arguments) =>
+          arguments.copy(
+            ledgerInboundMessageSizeMax = x
+        ))
+
       cmd("server")
         .text("serve data from platform")
         .action(

navigator/backend/src/main/scala/com/digitalasset/navigator/store/platform/PlatformStore.scala

@@ -46,7 +46,8 @@ object PlatformStore {
       platformPort: Int,
       tlsConfig: Option[TlsConfiguration],
       timeProviderType: TimeProviderType,
-      applicationInfo: ApplicationInfo
+      applicationInfo: ApplicationInfo,
+      ledgerMaxInbound: Int
   ): Props =
     Props(
       classOf[PlatformStore],
@@ -54,7 +55,8 @@ object PlatformStore {
       platformPort,
       tlsConfig,
       timeProviderType,
-      applicationInfo)
+      applicationInfo,
+      ledgerMaxInbound)
 
   type PlatformTime = Instant
 
@@ -81,7 +83,8 @@ class PlatformStore(
     platformPort: Int,
     tlsConfig: Option[TlsConfiguration],
     timeProviderType: TimeProviderType,
-    applicationInfo: ApplicationInfo
+    applicationInfo: ApplicationInfo,
+    ledgerMaxInbound: Int
 ) extends Actor
     with ActorLogging
     with Stash {
@@ -304,7 +307,7 @@ class PlatformStore(
 
     val builder = NettyChannelBuilder
       .forAddress(platformHost, platformPort)
-      .maxInboundMessageSize(50 * 1024 * 1024)
+      .maxInboundMessageSize(ledgerMaxInbound)
     configuration.sslContext match {
       case None => {
         log.info("Connecting to {}:{}, using a plaintext connection", platformHost, platformPort)