diff --git a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/Tests.scala b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/Tests.scala index a3974ce34e..54bf10bc35 100644 --- a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/Tests.scala +++ b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/Tests.scala @@ -69,25 +69,36 @@ object Tests { .map(BenchmarkReporter.toFile) .getOrElse(BenchmarkReporter.toStream(System.out)) .addReport(key, value) - Map( - PerformanceEnvelopeThroughputTestKey -> (new testtool.tests.PerformanceEnvelope.ThroughputTest( - logger = LoggerFactory.getLogger(PerformanceEnvelopeThroughputTestKey), - envelope = PerformanceEnvelope, - reporter = reporter, - )(_)), - PerformanceEnvelopeLatencyTestKey -> (new testtool.tests.PerformanceEnvelope.LatencyTest( - logger = LoggerFactory.getLogger(PerformanceEnvelopeLatencyTestKey), - envelope = PerformanceEnvelope, - reporter = reporter, - )(_)), - ) - } - private[this] val PerformanceEnvelope = Envelope.Beta // Should be adequate for most CIs + Envelope.values.flatMap { envelope => + { + val throughputKey: String = performanceEnvelopeThroughputTestKey(envelope) + val latencyKey: String = performanceEnvelopeLatencyTestKey(envelope) + List( + throughputKey -> (new testtool.tests.PerformanceEnvelope.ThroughputTest( + logger = LoggerFactory.getLogger(throughputKey), + envelope = envelope, + reporter = reporter, + )(_)), + latencyKey -> (new testtool.tests.PerformanceEnvelope.LatencyTest( + logger = LoggerFactory.getLogger(latencyKey), + envelope = envelope, + reporter = reporter, + )(_)), + ) + } + } + }.toMap - private[this] val PerformanceEnvelopeThroughputTestKey = "PerformanceEnvelope.Throughput" - private[this] val PerformanceEnvelopeLatencyTestKey = "PerformanceEnvelope.Latency" + private[this] def performanceEnvelopeThroughputTestKey(envelope: Envelope): String = + s"PerformanceEnvelope.${envelope.name}.Throughput" + private[this] def performanceEnvelopeLatencyTestKey(envelope: Envelope): String = + s"PerformanceEnvelope.${envelope.name}.Latency" private[testtool] val PerformanceTestsKeys = - Seq(PerformanceEnvelopeLatencyTestKey, PerformanceEnvelopeThroughputTestKey) + Envelope.values.flatMap { envelope => + List( + performanceEnvelopeThroughputTestKey(envelope), + performanceEnvelopeLatencyTestKey(envelope)) + } } diff --git a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/tests/PerformanceEnvelope.scala b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/tests/PerformanceEnvelope.scala index 8f4ea51baf..8971dd0bf0 100644 --- a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/tests/PerformanceEnvelope.scala +++ b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/tests/PerformanceEnvelope.scala @@ -37,26 +37,44 @@ import scala.util.{Failure, Success, Try} import com.daml.ledger.test.performance.{PingPong => PingPongModule} import org.slf4j.Logger -case class Envelope(name: String, transactionSizeKb: Int, throughput: Int, latencyMs: Int) +sealed trait Envelope { + val name: String + val transactionSizeKb: Int + val throughput: Int + val latencyMs: Int +} object Envelope { /** test will unlikely fail */ - object ProofOfConcept - extends Envelope("PoC", transactionSizeKb = 1, throughput = 0, latencyMs = 60000) + case object ProofOfConcept extends Envelope { + val name = "PoC"; val transactionSizeKb = 1; val throughput = 0; val latencyMs = 60000 + } /** test will fail if performance is lower than alpha envelope */ - object Alpha extends Envelope("Alpha", transactionSizeKb = 100, throughput = 5, latencyMs = 3000) + case object Alpha extends Envelope { + val name = "Alpha"; val transactionSizeKb = 100; val throughput = 5; val latencyMs = 3000 + } /** test will fail if performance is lower then beta envelope */ - object Beta extends Envelope("Beta", transactionSizeKb = 1000, throughput = 20, latencyMs = 1000) + case object Beta extends Envelope { + val name = "Beta"; val transactionSizeKb = 1000; val throughput = 20; val latencyMs = 1000 + } - object Public - extends Envelope("Public", transactionSizeKb = 5000, throughput = 50, latencyMs = 1000) + case object Public extends Envelope { + val name = "Public"; val transactionSizeKb = 5000; val throughput = 50; val latencyMs = 1000 + } - object Enterprise - extends Envelope("Enterprise", transactionSizeKb = 25000, throughput = 500, latencyMs = 500) + case object Enterprise extends Envelope { + val name = "Enterprise" + val transactionSizeKb = 25000 + val throughput = 500 + val latencyMs = 500 + } + // [FT] Could use macros as in https://riptutorial.com/scala/example/26215/using-sealed-trait-and-case-objects-and-allvalues-macro + // or even by pulling in https://github.com/lloydmeta/enumeratum but https://github.com/bazelbuild/rules_scala/issues/445 + val values: List[Envelope] = List[Envelope](ProofOfConcept, Alpha, Beta, Public, Enterprise) } trait PerformanceEnvelope { diff --git a/ledger/ledger-on-memory/BUILD.bazel b/ledger/ledger-on-memory/BUILD.bazel index 638fd2923d..c85eff0b58 100644 --- a/ledger/ledger-on-memory/BUILD.bazel +++ b/ledger/ledger-on-memory/BUILD.bazel @@ -161,8 +161,8 @@ conformance_test( ], test_tool_args = [ "--verbose", - "--perf-tests=PerformanceEnvelope.Throughput", - "--perf-tests=PerformanceEnvelope.Latency", + "--perf-tests=PerformanceEnvelope.Beta.Throughput", + "--perf-tests=PerformanceEnvelope.Beta.Latency", ], ) diff --git a/ledger/ledger-on-sql/BUILD.bazel b/ledger/ledger-on-sql/BUILD.bazel index 752fa9111c..889fd1db02 100644 --- a/ledger/ledger-on-sql/BUILD.bazel +++ b/ledger/ledger-on-sql/BUILD.bazel @@ -261,8 +261,8 @@ da_scala_test_suite( tags = db.get("benchmark_performance_envelope_tags", []), test_tool_args = db.get("benchmark_performance_envelope_tags", []) + [ "--verbose", - "--perf-tests=PerformanceEnvelope.Throughput", - "--perf-tests=PerformanceEnvelope.Latency", + "--perf-tests=PerformanceEnvelope.Beta.Throughput", + "--perf-tests=PerformanceEnvelope.Beta.Latency", ], ), )