Simplify test evidence generator [DPP-996] (#13553)

CHANGELOG_BEGIN
CHANGELOG_END

ledger/test-evidence/BUILD.bazel

@@ -28,7 +28,6 @@ da_scala_library(
 
 da_scala_binary(
     name = "generator",
-    testonly = True,
     main_class = "com.daml.test.evidence.generator.Main",
     runtime_deps = [
         "//ledger/sandbox-on-x:sandbox-on-x-it-tests-lib",

ledger/test-evidence/src/main/scala/com/daml/test/evidence/generator/LedgerApiTestGeneratorSupport.scala

@@ -21,10 +21,10 @@ object LedgerApiTestGeneratorSupport {
       }
 
       suite.tests
-        .map { test => test.name -> test.tags }
-        .mapFilter { case (testName, testTags) =>
+        .map { test => (test.suite.name, test.description) -> test.tags }
+        .mapFilter { case ((testName, description), testTags) =>
           testTags.collectFirst { case testTag: TT =>
-            testEntry(suite.name, testName, testTag, false, testSuite)
+            testEntry(testName, description, testTag, false, testSuite)
           }
         }
     }

ledger/test-evidence/src/main/scala/com/daml/test/evidence/generator/Main.scala

@@ -4,30 +4,14 @@
 package com.daml.test.evidence.generator
 
 import better.files.File
-import com.daml.ledger.api.testtool.suites
-import com.daml.ledger.api.testtool.infrastructure.LedgerTestSuite
-import com.daml.test.evidence.generator.TestEntry.{ReliabilityTestEntry, SecurityTestEntry}
-import com.daml.test.evidence.generator.TestEntryCsvEncoder.{
-  ReliabilityTestEntryCsv,
-  SecurityTestEntryCsv,
-  TestEntryCsv,
-}
-import com.daml.test.evidence.tag.Reliability.{ReliabilityTest, ReliabilityTestSuite}
-import com.daml.test.evidence.tag.Security.{SecurityTest, SecurityTestSuite}
+import com.daml.test.evidence.generator.TestEntryCsvEncoder.{SecurityTestEntryCsv, TestEntryCsv}
 import io.circe.Encoder
 import io.circe.generic.auto._
 import io.circe.syntax._
-import org.scalatest.Suite
 import com.daml.test.evidence.scalatest.JsonCodec.SecurityJson._
-import com.daml.test.evidence.scalatest.JsonCodec.ReliabilityJson._
-import org.scalatest.daml.ScalaTestAdapter
-
-import scala.reflect.ClassTag
 
 object Main {
 
-  private def loadClasspath(): Option[String] = Some(System.getProperty("java.class.path"))
-
   private def writeEvidenceToJsonFile[TE: Encoder](fileName: String, entries: List[TE]): Unit = {
     println(s"Writing inventory to $fileName...")
     val path = File(fileName)
@@ -49,54 +33,19 @@ object Main {
     println(s"Wrote to $path")
   }
 
-  private def collectTestEvidence[TT: ClassTag, TS: ClassTag, TE](
-      scalaTestSuites: List[Suite],
-      ledgerApiSuites: List[LedgerTestSuite],
-      testEntry: (String, String, TT, Boolean, Option[TS]) => TE,
-  ): List[TE] =
-    List.empty
-      .concat(ScalaTestGeneratorSupport.testEntries(scalaTestSuites, testEntry))
-      .concat(LedgerApiTestGeneratorSupport.testEntries(ledgerApiSuites, testEntry))
-
   def main(args: Array[String]): Unit = {
-    val runpathList: List[String] = loadClasspath()
-      .map(_.split(":").toList)
-      .getOrElse(List.empty)
+    if (args.length == 2) {
+      val securityTestEntries = TestEntryLookup.securityTestEntries
+      val csvEntries = securityTestEntries.map(SecurityTestEntryCsv.apply)
+      val csvFileName = args(0)
+      val jsonFileName = args(1)
 
-    val ledgerApiTests = List()
-      .concat(suites.v1_14.default(timeoutScaleFactor = 0L))
-      .concat(suites.v1_14.optional(tlsConfig = None))
-
-    val testSuites: List[Suite] = ScalaTestAdapter.loadTestSuites(runpathList)
-
-    println("Writing security tests inventory..")
-
-    val securityTestEntries =
-      collectTestEvidence[SecurityTest, SecurityTestSuite, SecurityTestEntry](
-        testSuites,
-        ledgerApiTests,
-        SecurityTestEntry,
+      writeEvidenceToCsvFile(csvFileName, csvEntries)
+      writeEvidenceToJsonFile(jsonFileName, securityTestEntries)
+    } else {
+      throw new IllegalArgumentException(
+        s"Invalid number of arguments, was ${args.length}, should be 2"
       )
-
-    val reliabilityTestEntries =
-      collectTestEvidence[ReliabilityTest, ReliabilityTestSuite, ReliabilityTestEntry](
-        testSuites,
-        ledgerApiTests,
-        ReliabilityTestEntry,
-      )
-
-    writeEvidenceToJsonFile("security-tests.json", securityTestEntries)
-    writeEvidenceToCsvFile(
-      "security-tests.csv",
-      securityTestEntries.map(SecurityTestEntryCsv.apply),
-    )
-
-    writeEvidenceToJsonFile("reliability-tests.json", reliabilityTestEntries)
-    writeEvidenceToCsvFile(
-      "reliability-tests.csv",
-      reliabilityTestEntries.map(ReliabilityTestEntryCsv.apply),
-    )
-
-    sys.exit()
+    }
   }
 }

ledger/test-evidence/src/main/scala/com/daml/test/evidence/generator/TestEntryLookup.scala

@@ -0,0 +1,59 @@
+// Copyright (c) 2022 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package com.daml.test.evidence.generator
+
+import com.daml.ledger.api.testtool.infrastructure.LedgerTestSuite
+import com.daml.ledger.api.testtool.suites
+import com.daml.test.evidence.generator.TestEntry.{ReliabilityTestEntry, SecurityTestEntry}
+import com.daml.test.evidence.tag.Reliability.{ReliabilityTest, ReliabilityTestSuite}
+import com.daml.test.evidence.tag.Security.{SecurityTest, SecurityTestSuite}
+import org.scalatest.Suite
+import org.scalatest.daml.ScalaTestAdapter
+
+import scala.reflect.ClassTag
+
+object TestEntryLookup {
+
+  private def collectTestEvidence[TT: ClassTag, TS: ClassTag, TE](
+      scalaTestSuites: List[Suite],
+      ledgerApiSuites: List[LedgerTestSuite],
+      testEntry: (String, String, TT, Boolean, Option[TS]) => TE,
+  ): List[TE] =
+    List.empty
+      .concat(ScalaTestGeneratorSupport.testEntries(scalaTestSuites, testEntry))
+      .concat(LedgerApiTestGeneratorSupport.testEntries(ledgerApiSuites, testEntry))
+
+  private def loadClasspath(): Option[String] = Option(System.getProperty("java.class.path"))
+
+  private def collectEntries[TT: ClassTag, TS: ClassTag, TE](
+      testEntry: (String, String, TT, Boolean, Option[TS]) => TE
+  ): List[TE] = {
+    val runpathList: List[String] = loadClasspath()
+      .map(_.split(":").toList)
+      .getOrElse(List.empty)
+
+    val ledgerApiTests = List()
+      .concat(suites.v1_14.default(timeoutScaleFactor = 0L))
+      .concat(suites.v1_14.optional(tlsConfig = None))
+
+    val testSuites: List[Suite] = ScalaTestAdapter.loadTestSuites(runpathList)
+
+    collectTestEvidence[TT, TS, TE](
+      testSuites,
+      ledgerApiTests,
+      testEntry,
+    )
+  }
+
+  def securityTestEntries: List[SecurityTestEntry] =
+    collectEntries[SecurityTest, SecurityTestSuite, SecurityTestEntry](
+      SecurityTestEntry.apply
+    ).sorted
+
+  def reliabilityTestEntries: List[ReliabilityTestEntry] =
+    collectEntries[ReliabilityTest, ReliabilityTestSuite, ReliabilityTestEntry](
+      ReliabilityTestEntry.apply
+    ).sorted
+
+}

libs-scala/test-evidence/generator/src/main/scala/com/daml/test/evidence/generator/TestEntry.scala

@@ -34,6 +34,13 @@ object TestEntry {
       suite: Option[SecurityTestSuite],
   ) extends TestEntry[SecurityTest, SecurityTestSuite]
 
+  object SecurityTestEntry {
+    implicit val ordering: Ordering[SecurityTestEntry] =
+      Ordering[(String, String, Int)].on[SecurityTestEntry] { entry =>
+        (entry.suiteName, entry.tag.file, entry.tag.line)
+      }
+  }
+
   final case class ReliabilityTestEntry(
       suiteName: String,
       description: String,
@@ -41,4 +48,11 @@ object TestEntry {
       ignored: Boolean,
       suite: Option[ReliabilityTestSuite],
   ) extends TestEntry[ReliabilityTest, ReliabilityTestSuite]
+
+  object ReliabilityTestEntry {
+    implicit val ordering: Ordering[ReliabilityTestEntry] =
+      Ordering[(String, String, Int)].on[ReliabilityTestEntry] { entry =>
+        (entry.suiteName, entry.tag.file, entry.tag.line)
+      }
+  }
 }