Canton conformance test debug logging and reduced test parallelism (#7322)

* Canton conformance test debug logging and reduced test parallelism

To reign in flakiness of a handful of TransactionServiceIT tests
that had run into timeouts during synchronize

Closes #7316

changelog_begin
changelog_end

* Review feedback from Stefano

ledger/ledger-api-test-tool-on-canton/BUILD.bazel

@@ -54,6 +54,7 @@ conformance_test(
         ":bootstrap.canton",
         ":canton",
         ":canton.conf",
+        ":logback-debug.xml",
         "@coreutils_nix//:bin/base64",
         "@curl_nix//:bin/curl",
         "@grpcurl_nix//:bin/grpcurl",
@@ -69,18 +70,10 @@ conformance_test(
     server = ":canton-test-runner-with-dependencies",
     test_tool_args = [
         "--verbose",
+        "--concurrent-test-runs=4",  # lowered from default #procs to reduce flakes - details in https://github.com/digital-asset/daml/issues/7316
         "--exclude=ContractKeysIT" +  # 3 of the more specialized tests currently fail on Canton
         ",ConfigManagementServiceIT,LedgerConfigurationServiceIT" +  # dynamic config management not supported by Canton
         ",TransactionScaleIT,LotsOfPartiesIT" +  # scale tests need timeout-scale-factor 1.5 and 2.0 respectively on Canton
-        ",ClosedWorldIT" +  # Canton currently fails this test with a different error (missing namespace in "unallocated" party id)
-        # FIXME https://github.com/digital-asset/daml/issues/7316 for the TransactionServiceIT flakes:
-        ",TransactionServiceIT:TXInvisibleTransactionTreeByEventId" +
-        ",TransactionServiceIT:TXFlatTransactionByIdSameAsTransactionStream" +
-        ",TransactionServiceIT:TXTransactionTreeByIdSameAsTransactionStream" +
-        ",TransactionServiceIT:TXSingleMultiSameTreesStakeholders" +
-        ",TransactionServiceIT:TXSingleMultiSameStakeholders",
+        ",ClosedWorldIT",  # Canton currently fails this test with a different error (missing namespace in "unallocated" party id)
     ],
-    # temporarily disabling canton tests, because the api server
-    # in the canton release still requires old time model fields.
-    # Enable the tests again, once canton is upgraded.
 ) if not is_windows else None

ledger/ledger-api-test-tool-on-canton/canton-test-runner.sh

@@ -9,7 +9,6 @@ set -o pipefail
 CANTON_COMMAND=(
   "$(rlocation com_github_digital_asset_daml/ledger/ledger-api-test-tool-on-canton/canton)"
   daemon
-  "-d"
   "--config=$(rlocation com_github_digital_asset_daml/ledger/ledger-api-test-tool-on-canton/canton.conf)"
   "--bootstrap=$(rlocation com_github_digital_asset_daml/ledger/ledger-api-test-tool-on-canton/bootstrap.canton)"
 )
@@ -64,6 +63,7 @@ HOME="$(mktemp -d)"
 export HOME
 # ammonite calls `System.getProperty('user.home')` which does not read $HOME.
 command+=("--wrapper_script_flag=--jvm_flag=-Duser.home=$HOME")
+command+=("--wrapper_script_flag=--jvm_flag=-Dlogback.configurationFile=$(rlocation com_github_digital_asset_daml/ledger/ledger-api-test-tool-on-canton/logback-debug.xml)")
 
 echo >&2 'Starting Canton...'
 "${command[@]}" &

ledger/ledger-api-test-tool-on-canton/logback-debug.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<configuration debug="false">
+  <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder>
+      <!-- attempt to place the correlation-id with a trailing space, however replace with an empty string if it's empty -->
+      <pattern>%date [%thread] %-5level %logger{10} %replace(cid:%mdc{correlation-id} ){'cid: ', ''}- %msg%n</pattern>
+    </encoder>
+  </appender>
+
+  <root level="INFO">
+    <appender-ref ref="STDOUT" />
+  </root>
+
+  <!-- Note that com.daml logs valuable info at TRACE level.
+       Set to TRACE for troubleshooting.
+  -->
+  <logger name="com.daml" level="DEBUG"/>
+  <!-- canton logs left at INFO to keep in check log file length and readability to "general daml-ledger" audience -->
+  <logger name="com.digitalasset" level="INFO"/>
+  <!-- netty reports ssl errors on DEBUG level ... -->
+  <logger name="io.netty.handler.ssl" level="INFO"/>
+</configuration>