ParticipantPruningIT added events for canton safe-pruning and multi-node synchronization (#10775)

* Hardening ParticipantPruningIT divulgence tests to not flake on canton-enterprise

worked out with @tudor-da

CHANGELOG_BEGIN
CHANGELOG_END

* Also feed other participant events

* Switch canton conformance tests to parallel indexer

* More stability fixes

ledger/ledger-api-test-tool-on-canton/BUILD.bazel

@@ -188,6 +188,7 @@ conformance_test(
     test_tool_args = [
         "--verbose",
         "--concurrent-test-runs=1",  # lowered from default #procs to reduce flakes - details in https://github.com/digital-asset/daml/issues/7316
+        "--timeout-scale-factor=4",  # provide additional time to avoid timing on on longer tests such as PRLocalAndNonLocalRetroactiveDivulgences or PRImmediateAndRetroactiveDivulgence
         "--include=ParticipantPruningIT",
     ],
 ) if not is_windows else None

ledger/ledger-api-test-tool-on-canton/enable-faster-pruning.conf

@@ -1,3 +1,15 @@
 canton.domains.test_domain.domain-parameters.reconciliation-interval = 2s
 canton.domains.test_domain.domain-parameters.participant-response-timeout = 2s
 canton.domains.test_domain.domain-parameters.mediator-reaction-timeout = 2s
+canton.participants.participant_1.ledger-api.enable-append-only-schema = true
+canton.participants.participant_1.ledger-api.enable-mutable-contract-state-cache = true
+canton.participants.participant_1.ledger-api.enable-in-memory-fan-out-for-ledger-api = true
+canton.participants.participant_2.ledger-api.enable-append-only-schema = true
+canton.participants.participant_2.ledger-api.enable-mutable-contract-state-cache = true
+canton.participants.participant_2.ledger-api.enable-in-memory-fan-out-for-ledger-api = true
+canton.participants.participant_3.ledger-api.enable-append-only-schema = true
+canton.participants.participant_3.ledger-api.enable-mutable-contract-state-cache = true
+canton.participants.participant_3.ledger-api.enable-in-memory-fan-out-for-ledger-api = true
+canton.participants.participant_4.ledger-api.enable-append-only-schema = true
+canton.participants.participant_4.ledger-api.enable-mutable-contract-state-cache = true
+canton.participants.participant_4.ledger-api.enable-in-memory-fan-out-for-ledger-api = true

ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ParticipantPruningIT.scala

@@ -8,6 +8,7 @@ import java.util.regex.Pattern
 import com.daml.ledger.api.testtool.infrastructure.Allocation._
 import com.daml.ledger.api.testtool.infrastructure.Assertions._
 import com.daml.ledger.api.testtool.infrastructure.LedgerTestSuite
+import com.daml.ledger.api.testtool.infrastructure.Synchronize.synchronize
 import com.daml.ledger.api.testtool.infrastructure.participant.ParticipantTestContext
 import com.daml.ledger.api.v1.ledger_offset.LedgerOffset
 import com.daml.ledger.api.v1.transaction.TransactionTree
@@ -22,7 +23,8 @@ import scala.concurrent.{ExecutionContext, Future}
 
 class ParticipantPruningIT extends LedgerTestSuite {
 
-  private val batchesToPopulate = 74
+  private val batchesToPopulate =
+    74 // One point of populating the ledger with a lot of events is to help advance canton's safe-pruning offsets
   private val lastItemToPruneIndex = batchesToPopulate
 
   test(
@@ -47,7 +49,7 @@ class ParticipantPruningIT extends LedgerTestSuite {
     for {
       cannotPruneNonHexOffset <- participant
         .prune("covfefe", attempts = 1, pruneAllDivulgedContracts = true)
-        .mustFail("pruning, specifiying a non-hexadecimal offset")
+        .mustFail("pruning, specifying a non-hexadecimal offset")
     } yield {
       assertGrpcError(
         cannotPruneNonHexOffset,
@@ -68,7 +70,7 @@ class ParticipantPruningIT extends LedgerTestSuite {
       actualEndExclusive <- participant.currentEnd()
       cannotPruneOffsetBeyondEnd <- participant
         .prune(actualEndExclusive, attempts = 1)
-        .mustFail("pruning, specifiying an offset after the ledger end")
+        .mustFail("pruning, specifying an offset after the ledger end")
     } yield {
       assertGrpcError(
         cannotPruneOffsetBeyondEnd,
@@ -440,6 +442,9 @@ class ParticipantPruningIT extends LedgerTestSuite {
       offsetToPruneUpToInSecondRealPrune = offsets((lastItemToPruneIndex + 1) * 2 - 1)
       offsetOfFirstSurvivingTransactionInSecondPrune = offsets((lastItemToPruneIndex + 1) * 2)
 
+      // Add more events before second prune too to advance canton's safe pruning offset
+      offsetsFollowingSecondRealPrune <- populateLedgerAndGetOffsets(participant, submitter)
+
       _ <- participant.prune(offsetToPruneUpToInSecondRealPrune)
 
       transactionsAfterSecondPrune <- participant.transactionTrees(
@@ -466,7 +471,7 @@ class ParticipantPruningIT extends LedgerTestSuite {
         s"transaction trees not pruned at expected offset after redundant prune",
       )
       assert(
-        transactionsAfterSecondPrune.size == offsets.size - 2 * (lastItemToPruneIndex + 1),
+        transactionsAfterSecondPrune.size == offsets.size - 2 * (lastItemToPruneIndex + 1) + offsetsFollowingSecondRealPrune.size,
         s"transaction tree count after second pruning does not match expected count",
       )
       assert(
@@ -597,6 +602,8 @@ class ParticipantPruningIT extends LedgerTestSuite {
       divulgence <- createDivulgence(alice, bob, alpha, beta)
       contract <- alpha.create(alice, Contract(alice))
 
+      _ <- synchronize(alpha, beta) // because of exercise on beta inside createDivulgence
+
       // Retroactively divulge Alice's contract to bob
       _ <- alpha.exercise(
         alice,
@@ -657,6 +664,7 @@ class ParticipantPruningIT extends LedgerTestSuite {
   )(implicit ec: ExecutionContext) =
     for {
       divulgenceHelper <- alpha.create(alice, DivulgenceProposal(alice, bob))
+      _ <- synchronize(alpha, beta)
       divulgence <- beta.exerciseAndGetContract[Divulgence](bob, divulgenceHelper.exerciseAccept)
     } yield divulgence
 
@@ -669,6 +677,8 @@ class ParticipantPruningIT extends LedgerTestSuite {
       divulgence: binding.Primitive.ContractId[Divulgence],
   )(implicit ec: ExecutionContext) =
     for {
+      _ <- synchronize(alpha, beta)
+
       // Check that Bob can fetch the contract
       _ <- beta.exerciseAndGetContract[Dummy](
         bob,
@@ -683,6 +693,8 @@ class ParticipantPruningIT extends LedgerTestSuite {
         divulgence.exerciseDivulge(_, contract),
       )
 
+      _ <- synchronize(alpha, beta)
+
       // Check that Bob can fetch the contract
       _ <- beta.exerciseAndGetContract[Dummy](
         bob,
@@ -699,6 +711,9 @@ class ParticipantPruningIT extends LedgerTestSuite {
         divulgence.exerciseCanFetch(_, contract),
       )
 
+      // Populate "other" participant too to advance canton's safe pruning offset
+      _ <- populateLedgerAndGetOffsets(alpha, alice)
+
       _ <- pruneAtCurrentOffset(beta, bob, pruneAllDivulgedContracts = true)
 
       _ <- beta
@@ -762,6 +777,9 @@ class ParticipantPruningIT extends LedgerTestSuite {
   )(implicit ec: ExecutionContext): Future[Unit] =
     for {
       offset <- participant.currentEnd()
+
+      _ <- populateLedgerAndGetOffsets(participant, localParty)
+
       // Dummy needed to prune at this offset
       _ <- participant.create(localParty, Dummy(localParty))