mirror of
https://github.com/digital-asset/daml.git
synced 2024-09-20 01:07:18 +03:00
[TestTool] Provide performance tests for all performance envelopes (#5473)
* Produce performance tests for all envelopes CHANGELOG_BEGIN [TestTool] Provide performance tests for all performance envelopes CHANGELOG_END * Follow Scala's recommended pattern for enum-like hierarchies
This commit is contained in:
parent
8d56ed3a7e
commit
1bb1517c08
@ -69,25 +69,36 @@ object Tests {
|
|||||||
.map(BenchmarkReporter.toFile)
|
.map(BenchmarkReporter.toFile)
|
||||||
.getOrElse(BenchmarkReporter.toStream(System.out))
|
.getOrElse(BenchmarkReporter.toStream(System.out))
|
||||||
.addReport(key, value)
|
.addReport(key, value)
|
||||||
Map(
|
|
||||||
PerformanceEnvelopeThroughputTestKey -> (new testtool.tests.PerformanceEnvelope.ThroughputTest(
|
|
||||||
logger = LoggerFactory.getLogger(PerformanceEnvelopeThroughputTestKey),
|
|
||||||
envelope = PerformanceEnvelope,
|
|
||||||
reporter = reporter,
|
|
||||||
)(_)),
|
|
||||||
PerformanceEnvelopeLatencyTestKey -> (new testtool.tests.PerformanceEnvelope.LatencyTest(
|
|
||||||
logger = LoggerFactory.getLogger(PerformanceEnvelopeLatencyTestKey),
|
|
||||||
envelope = PerformanceEnvelope,
|
|
||||||
reporter = reporter,
|
|
||||||
)(_)),
|
|
||||||
)
|
|
||||||
}
|
|
||||||
|
|
||||||
private[this] val PerformanceEnvelope = Envelope.Beta // Should be adequate for most CIs
|
Envelope.values.flatMap { envelope =>
|
||||||
|
{
|
||||||
|
val throughputKey: String = performanceEnvelopeThroughputTestKey(envelope)
|
||||||
|
val latencyKey: String = performanceEnvelopeLatencyTestKey(envelope)
|
||||||
|
List(
|
||||||
|
throughputKey -> (new testtool.tests.PerformanceEnvelope.ThroughputTest(
|
||||||
|
logger = LoggerFactory.getLogger(throughputKey),
|
||||||
|
envelope = envelope,
|
||||||
|
reporter = reporter,
|
||||||
|
)(_)),
|
||||||
|
latencyKey -> (new testtool.tests.PerformanceEnvelope.LatencyTest(
|
||||||
|
logger = LoggerFactory.getLogger(latencyKey),
|
||||||
|
envelope = envelope,
|
||||||
|
reporter = reporter,
|
||||||
|
)(_)),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}.toMap
|
||||||
|
|
||||||
private[this] val PerformanceEnvelopeThroughputTestKey = "PerformanceEnvelope.Throughput"
|
private[this] def performanceEnvelopeThroughputTestKey(envelope: Envelope): String =
|
||||||
private[this] val PerformanceEnvelopeLatencyTestKey = "PerformanceEnvelope.Latency"
|
s"PerformanceEnvelope.${envelope.name}.Throughput"
|
||||||
|
private[this] def performanceEnvelopeLatencyTestKey(envelope: Envelope): String =
|
||||||
|
s"PerformanceEnvelope.${envelope.name}.Latency"
|
||||||
|
|
||||||
private[testtool] val PerformanceTestsKeys =
|
private[testtool] val PerformanceTestsKeys =
|
||||||
Seq(PerformanceEnvelopeLatencyTestKey, PerformanceEnvelopeThroughputTestKey)
|
Envelope.values.flatMap { envelope =>
|
||||||
|
List(
|
||||||
|
performanceEnvelopeThroughputTestKey(envelope),
|
||||||
|
performanceEnvelopeLatencyTestKey(envelope))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
@ -37,26 +37,44 @@ import scala.util.{Failure, Success, Try}
|
|||||||
import com.daml.ledger.test.performance.{PingPong => PingPongModule}
|
import com.daml.ledger.test.performance.{PingPong => PingPongModule}
|
||||||
import org.slf4j.Logger
|
import org.slf4j.Logger
|
||||||
|
|
||||||
case class Envelope(name: String, transactionSizeKb: Int, throughput: Int, latencyMs: Int)
|
sealed trait Envelope {
|
||||||
|
val name: String
|
||||||
|
val transactionSizeKb: Int
|
||||||
|
val throughput: Int
|
||||||
|
val latencyMs: Int
|
||||||
|
}
|
||||||
|
|
||||||
object Envelope {
|
object Envelope {
|
||||||
|
|
||||||
/** test will unlikely fail */
|
/** test will unlikely fail */
|
||||||
object ProofOfConcept
|
case object ProofOfConcept extends Envelope {
|
||||||
extends Envelope("PoC", transactionSizeKb = 1, throughput = 0, latencyMs = 60000)
|
val name = "PoC"; val transactionSizeKb = 1; val throughput = 0; val latencyMs = 60000
|
||||||
|
}
|
||||||
|
|
||||||
/** test will fail if performance is lower than alpha envelope */
|
/** test will fail if performance is lower than alpha envelope */
|
||||||
object Alpha extends Envelope("Alpha", transactionSizeKb = 100, throughput = 5, latencyMs = 3000)
|
case object Alpha extends Envelope {
|
||||||
|
val name = "Alpha"; val transactionSizeKb = 100; val throughput = 5; val latencyMs = 3000
|
||||||
|
}
|
||||||
|
|
||||||
/** test will fail if performance is lower then beta envelope */
|
/** test will fail if performance is lower then beta envelope */
|
||||||
object Beta extends Envelope("Beta", transactionSizeKb = 1000, throughput = 20, latencyMs = 1000)
|
case object Beta extends Envelope {
|
||||||
|
val name = "Beta"; val transactionSizeKb = 1000; val throughput = 20; val latencyMs = 1000
|
||||||
|
}
|
||||||
|
|
||||||
object Public
|
case object Public extends Envelope {
|
||||||
extends Envelope("Public", transactionSizeKb = 5000, throughput = 50, latencyMs = 1000)
|
val name = "Public"; val transactionSizeKb = 5000; val throughput = 50; val latencyMs = 1000
|
||||||
|
}
|
||||||
|
|
||||||
object Enterprise
|
case object Enterprise extends Envelope {
|
||||||
extends Envelope("Enterprise", transactionSizeKb = 25000, throughput = 500, latencyMs = 500)
|
val name = "Enterprise"
|
||||||
|
val transactionSizeKb = 25000
|
||||||
|
val throughput = 500
|
||||||
|
val latencyMs = 500
|
||||||
|
}
|
||||||
|
|
||||||
|
// [FT] Could use macros as in https://riptutorial.com/scala/example/26215/using-sealed-trait-and-case-objects-and-allvalues-macro
|
||||||
|
// or even by pulling in https://github.com/lloydmeta/enumeratum but https://github.com/bazelbuild/rules_scala/issues/445
|
||||||
|
val values: List[Envelope] = List[Envelope](ProofOfConcept, Alpha, Beta, Public, Enterprise)
|
||||||
}
|
}
|
||||||
|
|
||||||
trait PerformanceEnvelope {
|
trait PerformanceEnvelope {
|
||||||
|
@ -161,8 +161,8 @@ conformance_test(
|
|||||||
],
|
],
|
||||||
test_tool_args = [
|
test_tool_args = [
|
||||||
"--verbose",
|
"--verbose",
|
||||||
"--perf-tests=PerformanceEnvelope.Throughput",
|
"--perf-tests=PerformanceEnvelope.Beta.Throughput",
|
||||||
"--perf-tests=PerformanceEnvelope.Latency",
|
"--perf-tests=PerformanceEnvelope.Beta.Latency",
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -261,8 +261,8 @@ da_scala_test_suite(
|
|||||||
tags = db.get("benchmark_performance_envelope_tags", []),
|
tags = db.get("benchmark_performance_envelope_tags", []),
|
||||||
test_tool_args = db.get("benchmark_performance_envelope_tags", []) + [
|
test_tool_args = db.get("benchmark_performance_envelope_tags", []) + [
|
||||||
"--verbose",
|
"--verbose",
|
||||||
"--perf-tests=PerformanceEnvelope.Throughput",
|
"--perf-tests=PerformanceEnvelope.Beta.Throughput",
|
||||||
"--perf-tests=PerformanceEnvelope.Latency",
|
"--perf-tests=PerformanceEnvelope.Beta.Latency",
|
||||||
],
|
],
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
Loading…
Reference in New Issue
Block a user