digital-asset/daml
1
1
Fork 0
mirror of https://github.com/digital-asset/daml.git synced 2024-11-13 00:16:19 +03:00
Code Issues Projects Releases Wiki Activity

Open-sourcing Gatling statistics reporter (#7325)

Browse Source 
* Open sourcing gatling statistics reporter

Running gatling scenarios with `RunLikeGatling` from libs-scala/gatling-utils

* cleaning up

* Replace "\n" with System.lineSeparator

so the formatting test cases pass on windows

* Testing DurationStatistics Monoid laws

* Renaming RunLikeGatling -> CustomRunner
This commit is contained in:
Leonid Shlyapnikov 2020-09-11 09:39:05 -04:00 committed by GitHub
parent f4f187b0ca
commit 4fbde3c672
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
25 changed files with 1355 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
ledger-service/http-json-perf/BUILD.bazel
View File

@ -5,12 +5,10 @@ load(
"//bazel_tools:scala.bzl",
"da_scala_binary",
"da_scala_library",
"da_scala_test",
"lf_scalacopts",
)
load("//rules_daml:daml.bzl", "daml_compile")
hj_scalacopts = lf_scalacopts + [
scalacopts = lf_scalacopts + [
"-P:wartremover:traverser:org.wartremover.warts.NonUnitStatements",
]
@ -20,7 +18,8 @@ da_scala_library(
plugins = [
"@maven//:org_spire_math_kind_projector_2_12",
],
scalacopts = hj_scalacopts,
resources = glob(["src/main/resources/**/*"]),
scalacopts = scalacopts,
tags = ["maven_coordinates=com.daml:http-json-perf:__VERSION__"],
visibility = ["//visibility:public"],
runtime_deps = [
@ -32,6 +31,7 @@ da_scala_library(
"//ledger-service/http-json",
"//ledger-service/http-json-testing",
"//ledger-service/jwt",
"//libs-scala/gatling-utils",
"//libs-scala/scala-utils",
"@maven//:com_fasterxml_jackson_core_jackson_core",
"@maven//:com_fasterxml_jackson_core_jackson_databind",
@ -59,7 +59,7 @@ da_scala_binary(
"-Dlogback.configurationFile=$(location :release/json-api-perf-logback.xml)",
],
main_class = "com.daml.http.perf.Main",
scalacopts = hj_scalacopts,
scalacopts = scalacopts,
tags = [
"maven_coordinates=com.daml:http-json-perf-deploy:__VERSION__",
"no_scala_version_suffix",
@ -67,8 +67,6 @@ da_scala_binary(
visibility = ["//visibility:public"],
runtime_deps = [
"@maven//:ch_qos_logback_logback_classic",
"@maven//:io_gatling_gatling_charts",
"@maven//:io_gatling_highcharts_gatling_highcharts",
],
deps = [
":http-json-perf",
@ -77,6 +75,7 @@ da_scala_binary(
"//ledger-service/http-json",
"//ledger-service/http-json-testing",
"//ledger-service/jwt",
"//libs-scala/gatling-utils",
"//libs-scala/scala-utils",
"@maven//:com_fasterxml_jackson_core_jackson_core",
"@maven//:com_fasterxml_jackson_core_jackson_databind",

128
ledger-service/http-json-perf/src/main/resources/gatling.conf Normal file
View File

@ -0,0 +1,128 @@
#########################
# Gatling Configuration #
#########################
# This file contains all the settings configurable for Gatling with their default values
gatling {
core {
#outputDirectoryBaseName = "" # The prefix for each simulation result folder (then suffixed by the report generation timestamp)
#runDescription = "" # The description for this simulation run, displayed in each report
#encoding = "utf-8" # Encoding to use throughout Gatling for file and string manipulation
#simulationClass = "" # The FQCN of the simulation to run (when used in conjunction with noReports, the simulation for which assertions will be validated)
#elFileBodiesCacheMaxCapacity = 200 # Cache size for request body EL templates, set to 0 to disable
#rawFileBodiesCacheMaxCapacity = 200 # Cache size for request body Raw templates, set to 0 to disable
#rawFileBodiesInMemoryMaxSize = 1000 # Below this limit, raw file bodies will be cached in memory
#pebbleFileBodiesCacheMaxCapacity = 200 # Cache size for request body Peeble templates, set to 0 to disable
#feederAdaptiveLoadModeThreshold = 100 # File size threshold (in MB). Below load eagerly in memory, above use batch mode with default buffer size
#shutdownTimeout = 10000 # Milliseconds to wait for the actor system to shutdown
extract {
regex {
#cacheMaxCapacity = 200 # Cache size for the compiled regexes, set to 0 to disable caching
}
xpath {
#cacheMaxCapacity = 200 # Cache size for the compiled XPath queries, set to 0 to disable caching
#preferJdk = false # When set to true, prefer JDK over Saxon for XPath-related operations
}
jsonPath {
#cacheMaxCapacity = 200 # Cache size for the compiled jsonPath queries, set to 0 to disable caching
}
css {
#cacheMaxCapacity = 200 # Cache size for the compiled CSS selectors queries, set to 0 to disable caching
}
}
directory {
#simulations = user-files/simulations # Directory where simulation classes are located (for bundle packaging only)
#resources = user-files/resources # Directory where resources, such as feeder files and request bodies are located (for bundle packaging only)
#reportsOnly = "" # If set, name of report folder to look for in order to generate its report
#binaries = "" # If set, name of the folder where compiles classes are located: Defaults to GATLING_HOME/target.
#results = results # Name of the folder where all reports folder are located
}
}
charting {
#noReports = false # When set to true, don't generate HTML reports
#maxPlotPerSeries = 1000 # Number of points per graph in Gatling reports
#useGroupDurationMetric = false # Switch group timings from cumulated response time to group duration.
indicators {
#lowerBound = 800 # Lower bound for the requests' response time to track in the reports and the console summary
#higherBound = 1200 # Higher bound for the requests' response time to track in the reports and the console summary
#percentile1 = 50 # Value for the 1st percentile to track in the reports, the console summary and Graphite
#percentile2 = 75 # Value for the 2nd percentile to track in the reports, the console summary and Graphite
#percentile3 = 95 # Value for the 3rd percentile to track in the reports, the console summary and Graphite
#percentile4 = 99 # Value for the 4th percentile to track in the reports, the console summary and Graphite
}
}
http {
#fetchedCssCacheMaxCapacity = 200 # Cache size for CSS parsed content, set to 0 to disable
#fetchedHtmlCacheMaxCapacity = 200 # Cache size for HTML parsed content, set to 0 to disable
#perUserCacheMaxCapacity = 200 # Per virtual user cache size, set to 0 to disable
#warmUpUrl = "https://gatling.io" # The URL to use to warm-up the HTTP stack (blank means disabled)
#enableGA = true # Very light Google Analytics, please support
ssl {
keyStore {
#type = "" # Type of SSLContext's KeyManagers store
#file = "" # Location of SSLContext's KeyManagers store
#password = "" # Password for SSLContext's KeyManagers store
#algorithm = "" # Algorithm used SSLContext's KeyManagers store
}
trustStore {
#type = "" # Type of SSLContext's TrustManagers store
#file = "" # Location of SSLContext's TrustManagers store
#password = "" # Password for SSLContext's TrustManagers store
#algorithm = "" # Algorithm used by SSLContext's TrustManagers store
}
}
ahc {
#connectTimeout = 10000 # Timeout in millis for establishing a TCP socket
#handshakeTimeout = 10000 # Timeout in millis for performing TLS handshake
#pooledConnectionIdleTimeout = 60000 # Timeout in millis for a connection to stay idle in the pool
#maxRetry = 2 # Number of times that a request should be tried again
#requestTimeout = 60000 # Timeout in millis for performing an HTTP request
#enableSni = true # When set to true, enable Server Name indication (SNI)
#enableHostnameVerification = false # When set to true, enable hostname verification: SSLEngine.setHttpsEndpointIdentificationAlgorithm("HTTPS")
#useInsecureTrustManager = true # Use an insecure TrustManager that trusts all server certificates
#sslEnabledProtocols = [] # Array of enabled protocols for HTTPS, if empty use Netty's defaults
#sslEnabledCipherSuites = [] # Array of enabled cipher suites for HTTPS, if empty enable all available ciphers
#sslSessionCacheSize = 0 # SSLSession cache size, set to 0 to use JDK's default
#sslSessionTimeout = 0 # SSLSession timeout in seconds, set to 0 to use JDK's default (24h)
#useOpenSsl = true # if OpenSSL should be used instead of JSSE
#useOpenSslFinalizers = false # if OpenSSL contexts should be freed with Finalizer or if using RefCounted is fine
#useNativeTransport = false # if native transport should be used instead of Java NIO (requires netty-transport-native-epoll, currently Linux only)
#enableZeroCopy = true # if zero-copy upload should be used if possible
#tcpNoDelay = true
#soKeepAlive = false # if TCP keepalive configured at OS level should be used
#soReuseAddress = false
#allocator = "pooled" # switch to unpooled for unpooled ByteBufAllocator
#maxThreadLocalCharBufferSize = 200000 # Netty's default is 16k
}
dns {
#queryTimeout = 5000 # Timeout in millis of each DNS query in millis
#maxQueriesPerResolve = 6 # Maximum allowed number of DNS queries for a given name resolution
}
}
jms {
#replyTimeoutScanPeriod = 1000 # scan period for timedout reply messages
}
data {
#writers = [console, file] # The list of DataWriters to which Gatling write simulation data (currently supported : console, file, graphite)
console {
#light = false # When set to true, displays a light version without detailed request stats
#writePeriod = 5 # Write interval, in seconds
}
file {
#bufferSize = 8192 # FileDataWriter's internal data buffer size, in bytes
}
leak {
#noActivityTimeout = 30 # Period, in seconds, for which Gatling may have no activity before considering a leak may be happening
}
graphite {
#light = false # only send the all* stats
#host = "localhost" # The host where the Carbon server is located
#port = 2003 # The port to which the Carbon server listens to (2003 is default for plaintext, 2004 is default for pickle)
#protocol = "tcp" # The protocol used to send data to Carbon (currently supported : "tcp", "udp")
#rootPathPrefix = "gatling" # The common prefix of all metrics sent to Graphite
#bufferSize = 8192 # Internal data buffer size, in bytes
#writePeriod = 1 # Write period, in seconds
}
}
}

33
ledger-service/http-json-perf/src/main/scala/com/daml/http/perf/Config.scala
View File

@ -7,15 +7,17 @@ import java.io.File
import com.daml.jwt.JwtDecoder
import com.daml.jwt.domain.Jwt
import scalaz.{Applicative, Traverse}
import scopt.RenderingMode
import scala.concurrent.duration.{Duration, FiniteDuration}
import scala.language.higherKinds
private[perf] final case class Config(
scenario: String,
private[perf] final case class Config[+S](
scenario: S,
dars: List[File],
jwt: Jwt,
reportsDir: Option[File],
reportsDir: File,
maxDuration: Option[FiniteDuration]
) {
override def toString: String =
@ -23,21 +25,34 @@ private[perf] final case class Config(
s"scenario=${this.scenario}, " +
s"dars=${dars: List[File]}," +
s"jwt=..., " + // don't print the JWT
s"reportsDir=${reportsDir: Option[File]}," +
s"reportsDir=${reportsDir: File}," +
s"maxDuration=${this.maxDuration: Option[FiniteDuration]}" +
")"
}
private[perf] object Config {
val Empty =
Config(scenario = "", dars = List.empty, jwt = Jwt(""), reportsDir = None, maxDuration = None)
Config[String](
scenario = "",
dars = List.empty,
jwt = Jwt(""),
reportsDir = new File(""),
maxDuration = None)
def parseConfig(args: Seq[String]): Option[Config] =
implicit val configInstance: Traverse[Config] = new Traverse[Config] {
override def traverseImpl[G[_]: Applicative, A, B](fa: Config[A])(
f: A => G[B]): G[Config[B]] = {
import scalaz.syntax.functor._
f(fa.scenario).map(b => Empty.copy(scenario = b))
}
}
def parseConfig(args: Seq[String]): Option[Config[String]] =
configParser.parse(args, Config.Empty)
@SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements"))
private val configParser: scopt.OptionParser[Config] =
new scopt.OptionParser[Config]("http-json-perf-binary") {
private val configParser: scopt.OptionParser[Config[String]] =
new scopt.OptionParser[Config[String]]("http-json-perf-binary") {
override def renderingMode: RenderingMode = RenderingMode.OneColumn
head("JSON API Perf Test Tool")
@ -61,7 +76,7 @@ private[perf] object Config {
.text("JWT token to use when connecting to JSON API.")
opt[File]("reports-dir")
.action((x, c) => c.copy(reportsDir = Some(x)))
.action((x, c) => c.copy(reportsDir = x))
.optional()
.text("Directory where reports generated. If not set, reports will not be generated.")

155
ledger-service/http-json-perf/src/main/scala/com/daml/http/perf/Main.scala
View File

@ -3,18 +3,26 @@
package com.daml.http.perf
import java.io.File
import akka.actor.ActorSystem
import akka.stream.Materializer
import com.daml.gatling.stats.SimulationLog.ScenarioStats
import com.daml.gatling.stats.{SimulationLog, SimulationLogSyntax}
import com.daml.grpc.adapter.{AkkaExecutionSequencerPool, ExecutionSequencerFactory}
import com.daml.http.HttpServiceTestFixture.withHttpService
import com.daml.http.domain.LedgerId
import com.daml.http.perf.scenario.SimulationConfig
import com.daml.http.util.FutureUtil._
import com.daml.http.{EndpointsCompanion, HttpService}
import com.daml.jwt.domain.Jwt
import com.daml.scalautil.Statement.discard
import com.typesafe.scalalogging.StrictLogging
import scalaz.\/
import io.gatling.core.scenario.Simulation
import scalaz.std.scalaFuture._
import scalaz.syntax.tag._
import scalaz.{-\/, EitherT, \/, \/-}
import scalaz.std.string._
import scala.concurrent.duration.{Duration, _}
import scala.concurrent.{Await, ExecutionContext, Future, TimeoutException}
@ -22,6 +30,8 @@ import scala.util.{Failure, Success}
object Main extends StrictLogging {
private type ET[A] = EitherT[Future, Throwable, A]
sealed abstract class ExitCode(val code: Int) extends Product with Serializable
object ExitCode {
case object Ok extends ExitCode(0)
@ -45,60 +55,93 @@ object Main extends StrictLogging {
def terminate(): Unit = discard { Await.result(asys.terminate(), terminationTimeout) }
val exitCode: ExitCode = Config.parseConfig(args) match {
case Some(config) =>
val exitCodeF: Future[ExitCode] = main(config)
exitCodeF.onComplete {
case Success(_) => logger.info(s"Scenario: ${config.scenario: String} completed")
case Failure(e) => logger.error(s"Scenario: ${config.scenario: String} failed", e)
}
try {
Await.result(exitCodeF, config.maxDuration.getOrElse(Duration.Inf))
} catch {
case e: TimeoutException =>
logger.error(s"Scenario: ${config.scenario: String} failed", e)
ExitCode.TimedOutScenario
}
case None =>
// error is printed out by scopt
ExitCode.InvalidUsage
case Some(config) =>
waitForResult(logCompletion(main1(config)), config.maxDuration.getOrElse(Duration.Inf))
}
terminate()
sys.exit(exitCode.code)
}
private def main(config: Config)(
private def logCompletion(fa: Future[Throwable \/ _])(implicit ec: ExecutionContext): fa.type = {
fa.onComplete {
case Success(\/-(_)) => logger.info(s"Scenario completed")
case Success(-\/(e)) => logger.error(s"Scenario failed", e)
case Failure(e) => logger.error(s"Scenario failed", e)
}
fa
}
private def waitForResult[A](fa: Future[Throwable \/ ExitCode], timeout: Duration): ExitCode =
try {
Await
.result(fa, timeout)
.valueOr(_ => ExitCode.GatlingError)
} catch {
case e: TimeoutException =>
logger.error(s"Scenario failed", e)
ExitCode.TimedOutScenario
}
private def main1(config: Config[String])(
implicit asys: ActorSystem,
mat: Materializer,
aesf: ExecutionSequencerFactory,
ec: ExecutionContext
): Future[ExitCode] = {
): Future[Throwable \/ ExitCode] = {
import scalaz.syntax.traverse._
logger.info(s"$config")
val ledgerId = getLedgerId(config.jwt)
.getOrElse(throw new IllegalArgumentException("Cannot infer Ledger ID from JWT"))
val et: ET[ExitCode] = for {
ledgerId <- either(
getLedgerId(config.jwt).leftMap(_ =>
new IllegalArgumentException("Cannot infer Ledger ID from JWT"))
): ET[LedgerId]
if (isValidScenario(config.scenario)) {
withHttpService(ledgerId.unwrap, config.dars, None, None) { (uri, _, _, _) =>
runGatlingScenario(config, uri.authority.host.address, uri.authority.port)
}
} else {
logger.error(s"Invalid scenario: ${config.scenario}")
Future.successful(ExitCode.InvalidScenario)
}
_ <- either(
config.traverse(s => resolveSimulationClass(s))
): ET[Config[Class[_ <: Simulation]]]
exitCode <- rightT(
main2(ledgerId, config)
): ET[ExitCode]
} yield exitCode
et.run
}
private def isValidScenario(scenario: String): Boolean = {
private def main2(ledgerId: LedgerId, config: Config[String])(
implicit asys: ActorSystem,
mat: Materializer,
aesf: ExecutionSequencerFactory,
ec: ExecutionContext
): Future[ExitCode] =
withHttpService(ledgerId.unwrap, config.dars, None, None) { (uri, _, _, _) =>
runGatlingScenario(config, uri.authority.host.address, uri.authority.port)
.flatMap {
case (exitCode, dir) =>
toFuture(generateReport(dir))
.map { _ =>
logger.info(s"Report directory: ${dir.getAbsolutePath}")
exitCode
}
}: Future[ExitCode]
}
private def resolveSimulationClass(str: String): Throwable \/ Class[_ <: Simulation] = {
try {
val klass: Class[_] = Class.forName(scenario)
classOf[io.gatling.core.scenario.Simulation].isAssignableFrom(klass)
val klass: Class[_] = Class.forName(str)
val simClass = klass.asSubclass(classOf[Simulation])
\/-(simClass)
} catch {
case e: ClassCastException =>
logger.error(s"Invalid Gatling scenario: '$scenario'", e)
false
case e: Throwable =>
logger.error(s"Cannot find Gatling scenario: '$scenario'", e)
false
logger.error(s"Cannot resolve scenario: '$str'", e)
-\/(e)
}
}
@ -107,8 +150,10 @@ object Main extends StrictLogging {
.decodeAndParsePayload(jwt, HttpService.decodeJwt)
.map { case (_, payload) => payload.ledgerId }
private def runGatlingScenario(config: Config, jsonApiHost: String, jsonApiPort: Int)(
implicit ec: ExecutionContext): Future[ExitCode] = {
private def runGatlingScenario(config: Config[String], jsonApiHost: String, jsonApiPort: Int)(
implicit sys: ActorSystem,
ec: ExecutionContext): Future[(ExitCode, File)] = {
import io.gatling.app
import io.gatling.core.config.GatlingPropertiesBuilder
@ -118,11 +163,41 @@ object Main extends StrictLogging {
val configBuilder = new GatlingPropertiesBuilder()
.simulationClass(config.scenario)
.resultsDirectory(config.reportsDir.map(_.getAbsolutePath).getOrElse("./reports"))
.noReports() // TODO(Leo): we want the reports, but they are currently failing with a runtime exception
.resultsDirectory(config.reportsDir.getAbsolutePath)
.noReports()
Future(app.Gatling.fromMap(configBuilder.build))
.map(a => if (a == app.cli.StatusCode.Success.code) ExitCode.Ok else ExitCode.GatlingError)
Future
.fromTry {
app.CustomRunner.runWith(sys, configBuilder.build, None)
}
.map {
case (a, f) =>
if (a == app.cli.StatusCode.Success.code) (ExitCode.Ok, f) else (ExitCode.GatlingError, f)
}
}
private def generateReport(dir: File): String \/ Unit = {
import SimulationLogSyntax._
require(dir.isDirectory)
val logPath = new File(dir, "simulation.log")
val simulationLog = SimulationLog.fromFile(logPath)
simulationLog.foreach { x =>
x.writeSummary(dir)
logger.info(s"Report\n${formatReport(x.scenarios)}")
}
simulationLog.map(_ => ())
}
private def formatReport(scenarios: List[ScenarioStats]): String = {
val buf = new StringBuffer()
scenarios.foreach { x =>
x.requestsByType.foreach {
case (name, stats) =>
buf.append(stats.formatted(name))
}
}
buf.toString
}
}

63
libs-scala/gatling-utils/BUILD.bazel Normal file
View File

@ -0,0 +1,63 @@
# Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
load(
"//bazel_tools:scala.bzl",
"da_scala_library",
"da_scala_test",
"lf_scalacopts",
)
scalacopts = lf_scalacopts + [
"-P:wartremover:traverser:org.wartremover.warts.NonUnitStatements",
]
da_scala_library(
name = "gatling-utils",
srcs = glob(["src/main/scala/**/*.scala"]),
plugins = [
"@maven//:org_spire_math_kind_projector_2_12",
],
scalacopts = scalacopts,
tags = ["maven_coordinates=com.daml:gatling-utils:__VERSION__"],
visibility = ["//visibility:public"],
runtime_deps = [
"@maven//:ch_qos_logback_logback_classic",
],
deps = [
"//libs-scala/scala-utils",
"@maven//:com_typesafe_akka_akka_actor_2_12",
"@maven//:com_typesafe_scala_logging_scala_logging_2_12",
"@maven//:io_gatling_gatling_app",
"@maven//:io_gatling_gatling_core",
"@maven//:org_scalaz_scalaz_core_2_12",
"@maven//:org_slf4j_slf4j_api",
],
)
filegroup(
name = "test-simulation-logs",
srcs = glob(["src/test/resources/simulation-log/*"]),
)
da_scala_test(
name = "tests",
size = "small",
srcs = glob(["src/test/scala/**/*.scala"]),
data = [
":test-simulation-logs",
],
plugins = [
"@maven//:org_spire_math_kind_projector_2_12",
],
scalacopts = scalacopts,
deps = [
":gatling-utils",
"//bazel_tools/runfiles:scala_runfiles",
"//libs-scala/scalatest-utils",
"@maven//:org_scalacheck_scalacheck_2_12",
"@maven//:org_scalatest_scalatest_2_12",
"@maven//:org_scalaz_scalaz_core_2_12",
"@maven//:org_scalaz_scalaz_scalacheck_binding_2_12",
],
)

19
libs-scala/gatling-utils/src/main/scala/com/daml/gatling/stats/Count.scala Normal file
View File

@ -0,0 +1,19 @@
// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package com.daml.gatling.stats
// generic container for counted Gatling things
// the T parameter is often Int, sometimes Double, Long for requestCount
case class Count[T](total: T, ok: T, ko: T) {
import OutputFormattingHelpers._
def formatted(name: String)(implicit ev: Numeric[T]): String =
s"> %-${available}s%8s (OK=%-6s KO=%-6s)".format(
name,
printN(total),
printN(ok),
printN(ko)
)
}

17
libs-scala/gatling-utils/src/main/scala/com/daml/gatling/stats/OutputFormattingHelpers.scala Normal file
View File

@ -0,0 +1,17 @@
// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package com.daml.gatling.stats
object OutputFormattingHelpers {
val lineLength = 80
val available = lineLength - 32
val formatter = new java.text.DecimalFormat("###.###")
def subtitle(title: String): String =
("---- " + title + " ").padTo(lineLength, '-')
def printN[N](value: N)(implicit N: Numeric[N]): String =
if (value == N.zero) "-" else formatter.format(value)
}

288
libs-scala/gatling-utils/src/main/scala/com/daml/gatling/stats/SimulationLog.scala Normal file
View File

@ -0,0 +1,288 @@
// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
// Copyright (c) 2019, Digital Asset (Switzerland) GmbH and/or its affiliates.
// All rights reserved.
package com.daml.gatling.stats
import java.io.File
import scalaz._
import Scalaz._
import scala.collection.immutable.ListMap
import com.daml.gatling.stats.util.NonEmptySyntax._
import com.daml.gatling.stats.util.ReadFileSyntax._
import com.daml.gatling.stats.OutputFormattingHelpers._
import SimulationLog._
case class SimulationLog(simulation: String, scenarios: List[ScenarioStats]) {
import Scalaz._
lazy val requestsByType: Map[String, RequestTypeStats] =
scenarios.foldRight(Map.empty[String, RequestTypeStats])(_.requestsByType |+| _)
def toCsvString: String =
toCsv
.foldRight(Option.empty[Seq[String]]) { (row, result) =>
result
.orElse(List(row.keys.mkString(",")).some)
.map(_ :+ row.values.map(_.toString.filterNot(_ == ',')).mkString(","))
}
.map(_.mkString("", System.lineSeparator(), System.lineSeparator()))
.getOrElse("")
private def toCsv: List[ListMap[String, String]] = {
scenarios
.flatMap { scenario =>
scenario.requestsByType.map {
case (requestType, stats) =>
ListMap(
"simulation" -> simulation.toString,
"scenario" -> scenario.label,
"maxUsers" -> scenario.maxUsers.toString,
"request" -> requestType,
"start" -> format(stats.successful.start),
"duration" -> format(stats.successful.duration.map(_.toDouble / 1000)),
"end" -> format(stats.successful.end),
"count" -> stats.count.toString,
"successCount" -> stats.successful.count.toString,
"errorCount" -> stats.failed.count.toString,
"min" -> format(stats.successful.percentile(0.0)),
"p90" -> format(stats.successful.percentile(0.9)),
"p95" -> format(stats.successful.percentile(0.95)),
"p99" -> format(stats.successful.percentile(0.99)),
"p999" -> format(stats.successful.percentile(0.999)),
"max" -> format(stats.successful.percentile(1.0)),
"mean" -> format(stats.successful.geometricMean.map(math.round)),
"avg" -> format(stats.successful.mean.map(math.round)),
"stddev" -> format(stats.successful.stdDev.map(math.round)),
"rps" -> format(stats.successful.requestsPerSecond)
)
}
}
}
}
object SimulationLog {
type Timestamp = Long
private def format[A: Numeric: Show](fa: Option[A]): String = {
import scalaz.syntax.show._
val num = implicitly[Numeric[A]]
fa.getOrElse(num.zero).shows
}
case class ScenarioStats(
label: String,
maxUsers: Int = 0,
requestsByType: Map[String, RequestTypeStats] = Map.empty)
case class DurationStatistics(
durations: Seq[Int],
start: Option[Timestamp],
end: Option[Timestamp]) {
def count: Int = durations.size
def mean: Option[Double] = durations.nonEmptyOpt.map(ds => ds.sum.toDouble / ds.size)
def geometricMean: Option[Double] =
durations.nonEmptyOpt.map(ds => math.exp(ds.map(d => math.log(d.toDouble)).sum / ds.size))
def duration: Option[Int] = for { s <- start; e <- end } yield (e - s).toInt
def requestsPerSecond: Option[Double] = duration.map(count.toDouble / _.toDouble * 1000)
def stdDev: Option[Double] =
for {
avg <- mean
variance <- durations.nonEmptyOpt.map(ds => ds.map(d => math.pow(d - avg, 2)).sum / ds.size)
} yield math.sqrt(variance)
def percentile(p: Double): Option[Int] = {
require(p >= 0.0 && p <= 1.0, "Percentile must be between zero and one, inclusive.")
sortedDurations.nonEmptyOpt.map(ds => ds(Math.round((ds.size - 1).toDouble * p).toInt))
}
private lazy val sortedDurations = durations.toIndexedSeq.sorted
}
object DurationStatistics {
implicit val durationStatisticsMonoid: Monoid[DurationStatistics] =
new Monoid[DurationStatistics] {
override def zero: DurationStatistics = DurationStatistics(Seq.empty, None, None)
override def append(s1: DurationStatistics, s2: => DurationStatistics): DurationStatistics =
DurationStatistics(
s1.durations ++ s2.durations,
min(s1.start, s2.start),
max(s1.end, s2.end)
)
}
private def min[A](fa: Option[A], fb: Option[A])(
implicit ev: scala.math.Ordering[A]): Option[A] = (fa, fb) match {
case (Some(x), Some(y)) => Some(ev.min(x, y))
case (None, x @ Some(_)) => x
case (x @ Some(_), None) => x
case (_, _) => None
}
private def max[A](fa: Option[A], fb: Option[A])(
implicit ev: scala.math.Ordering[A]): Option[A] = (fa, fb) match {
case (Some(x), Some(y)) => Some(ev.max(x, y))
case (None, x @ Some(_)) => x
case (x @ Some(_), None) => x
case (_, _) => None
}
}
case class RequestTypeStats(successful: DurationStatistics, failed: DurationStatistics) {
def count: Int = successful.count + failed.count
// takes a function that calculates a metric for DurationStatistics, and generates a Count for all/successful/failed
// based on that function
def attribute[T](f: DurationStatistics => Option[T])(implicit N: Numeric[T]): Count[T] =
Count(f(all).getOrElse(N.zero), f(successful).getOrElse(N.zero), f(failed).getOrElse(N.zero))
def durationGroup(from: Option[Int], to: Option[Int]) = {
val title = from.map(v => s"$v ms < ").getOrElse("") + "t" + to
.map(v => s" < $v ms")
.getOrElse("")
val count = successful.durations.count(d => !from.exists(d < _) && !to.exists(d >= _))
StatGroup(
title,
count,
all.durations.nonEmptyOpt.map(ds => count.toDouble / ds.size * 100).getOrElse(0.0))
}
def formatted(title: String): String =
List(
"=" * lineLength,
subtitle(title),
attribute(_.count.some).formatted("Number of requests"),
attribute(_.durations.nonEmptyOpt.map(_.min)).formatted("Min. response time"),
attribute(_.durations.nonEmptyOpt.map(_.max)).formatted("Max. response time"),
attribute(_.mean.map(math.round)).formatted("Mean response time"),
attribute(_.stdDev.map(math.round)).formatted("Std. deviation"),
attribute(_.percentile(0.9)).formatted("response time 90th percentile"),
attribute(_.percentile(0.95)).formatted("response time 95th percentile"),
attribute(_.percentile(0.99)).formatted("response time 99th percentile"),
attribute(_.percentile(0.999)).formatted("response time 99.9th percentile"),
attribute(_.requestsPerSecond).formatted("Mean requests/second"),
subtitle("Response time distribution"),
durationGroup(None, 5000.some).formatted,
durationGroup(5000.some, 30000.some).formatted,
durationGroup(30000.some, None).formatted,
StatGroup(
"failed",
failed.durations.size,
all.durations.nonEmptyOpt
.map(ds => failed.durations.size.toDouble / ds.size * 100)
.getOrElse(0.0)).formatted,
"=" * lineLength
).mkString(System.lineSeparator)
lazy val all = successful |+| failed
}
object RequestTypeStats {
def fromRequestStats(requests: Seq[RequestStats]): RequestTypeStats = {
val successful = Map(true -> Seq(), false -> Seq()) ++ requests.groupBy(_.successful)
val start = requests.nonEmptyOpt.map(_.map(_.start).min)
RequestTypeStats(
successful = DurationStatistics(
successful(true).map(_.duration),
start,
successful(true).map(_.end).nonEmptyOpt.map(_.max)),
failed = DurationStatistics(
successful(false).map(_.duration),
start,
successful(false).map(_.end).nonEmptyOpt.map(_.max))
)
}
implicit val requestTypeStatsMonoid: Monoid[RequestTypeStats] = new Monoid[RequestTypeStats] {
override def zero: RequestTypeStats =
RequestTypeStats(mzero[DurationStatistics], mzero[DurationStatistics])
override def append(s1: RequestTypeStats, s2: => RequestTypeStats): RequestTypeStats =
RequestTypeStats(s1.successful |+| s2.successful, s1.failed |+| s2.failed)
}
}
case class RequestStats(
userId: Int,
requestLabel: String,
start: Timestamp,
end: Timestamp,
successful: Boolean
) {
def duration: Int = (end - start).toInt
}
def fromFile(file: File): String \/ SimulationLog =
for {
content <- file.contentsAsString.leftMap(_.getMessage)
simulation <- fromString(content)
} yield simulation
def fromString(content: String): String \/ SimulationLog =
for {
rowsByType <- groupRowsByType(content)
requests <- processRequests(rowsByType.getOrElse("REQUEST", List.empty))
scenarios <- processScenarios(requests.groupBy(_.userId))(
rowsByType.getOrElse("USER", List.empty))
simulation <- processSimulation(scenarios)(rowsByType.getOrElse("RUN", List.empty))
} yield simulation
private def groupRowsByType(fileContent: String) =
\/.fromTryCatchNonFatal {
fileContent
.split('\n')
.map(_.trim.split('\t').toSeq)
.collect { case rowType +: fields => rowType -> fields }
.toList
.groupBy(_._1)
.mapValues(_.map(_._2))
}.leftMap(_.getMessage)
def processScenarios(requestsByUser: Map[Int, Seq[RequestStats]])(
userRows: List[Seq[String]]): String \/ List[ScenarioStats] =
if (userRows.isEmpty) "Could not find any USER rows.".left
else
userRows
.collect { case Seq(label, userId, "START", _*) => userId -> label }
.foldRight(Map.empty[String, ScenarioStats]) {
case ((userId, label), result) =>
val requestsByType = requestsByUser
.getOrElse(userId.toInt, Seq.empty)
.groupBy(_.requestLabel)
.mapValues(RequestTypeStats.fromRequestStats)
val s = result.getOrElse(label, ScenarioStats(label))
result + (label -> s.copy(
maxUsers = s.maxUsers + 1,
requestsByType = s.requestsByType |+| requestsByType))
}
.values
.toList
.right
private def processSimulation(scenarios: List[ScenarioStats])(
runRows: List[Seq[String]]): String \/ SimulationLog =
if (runRows.size != 1) s"Expected one RUN row in log, but found ${runRows.size}.".left
else
runRows.head match {
case Seq(_, simulation, _*) => SimulationLog(simulation, scenarios).right
case _ => "Found illegal RUN row.".left
}
private def processRequests(requestRows: List[Seq[String]]): String \/ Seq[RequestStats] =
requestRows.traverseU {
case Seq(userId, _, scenarioName, start, end, status, _*) =>
\/.fromTryCatchNonFatal( // .toInt/Long throws if column non-numeric
RequestStats(userId.toInt, scenarioName, start.toLong, end.toLong, status == "OK"))
.leftMap(_.getMessage)
case _ =>
"Received REQUEST row with illegal number of fields".left
}
}

27
libs-scala/gatling-utils/src/main/scala/com/daml/gatling/stats/SimulationLogSyntax.scala Normal file
View File

@ -0,0 +1,27 @@
// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package com.daml.gatling.stats
import java.io.File
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import com.daml.scalautil.Statement.discard
object SimulationLogSyntax {
implicit class SimulationLogOps(val log: SimulationLog) extends AnyVal {
/**
* Will write a summary.csv given a Gatling result directory.
* @param targetDirectory the directory where the summary.csv will be created.
*/
def writeSummary(targetDirectory: File): Unit = {
discard {
Files.write(
new File(targetDirectory, "summary.csv").toPath,
log.toCsvString.getBytes(StandardCharsets.UTF_8))
}
}
}
}

11
libs-scala/gatling-utils/src/main/scala/com/daml/gatling/stats/StatGroup.scala Normal file
View File

@ -0,0 +1,11 @@
// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package com.daml.gatling.stats
// generic container for Gatling statistics things
case class StatGroup(name: String, count: Int, percentage: Double) {
import OutputFormattingHelpers._
def formatted: String = s"> %-${available}s%8s (%3s%%)".format(name, count, printN(percentage))
}

10
libs-scala/gatling-utils/src/main/scala/com/daml/gatling/stats/util/NonEmptySyntax.scala Normal file
View File

@ -0,0 +1,10 @@
// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package com.daml.gatling.stats.util
object NonEmptySyntax {
implicit class NonEmptyOps[A](val seq: Seq[A]) extends AnyVal {
def nonEmptyOpt: Option[Seq[A]] = if (seq.isEmpty) None else Some(seq)
}
}

24
libs-scala/gatling-utils/src/main/scala/com/daml/gatling/stats/util/ReadFileSyntax.scala Normal file
View File

@ -0,0 +1,24 @@
// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package com.daml.gatling.stats.util
import java.io.File
import scalaz.\/
import scala.io.{BufferedSource, Source}
object ReadFileSyntax {
implicit class FileSourceOps(val path: File) extends AnyVal {
def contentsAsString: Throwable \/ String =
withSource(_.mkString)
def withSource(f: BufferedSource => String): Throwable \/ String =
\/.fromTryCatchNonFatal {
val source = Source.fromFile(path)
try f(source)
finally source.close()
}
}
}

44
libs-scala/gatling-utils/src/main/scala/io/gatling/app/CustomRunner.scala Normal file
View File

@ -0,0 +1,44 @@
// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package io.gatling.app
import java.io.File
import java.nio.file.FileSystems
import akka.actor.ActorSystem
import com.daml.scalautil.Statement.discard
import com.typesafe.scalalogging.StrictLogging
import io.gatling.core.config.GatlingConfiguration
import io.gatling.core.scenario.Simulation
import scala.util.Try
object CustomRunner extends StrictLogging {
// Copies the io.gatling.app.Gatling start method, which is Copyright 2011-2019
// GatlingCorp (https://gatling.io) under the Apache 2.0 license (http://www.apache.org/licenses/LICENSE-2.0)
// This derivation returns the results directory of the run for additional post-processing.
def runWith(
system: ActorSystem,
overrides: ConfigOverrides,
mbSimulation: Option[Class[Simulation]] = None
): Try[(Int, File)] = {
logger.trace("Starting")
// workaround for deadlock issue, see https://github.com/gatling/gatling/issues/3411
discard { FileSystems.getDefault }
val configuration = GatlingConfiguration.load(overrides)
logger.trace("Configuration loaded")
val runResult = Try {
Runner(system, configuration).run(mbSimulation)
}
runResult map { res =>
val status = new RunResultProcessor(configuration).processRunResult(res).code
(status, new File(configuration.core.directory.results, res.runId))
}
}
}

4
libs-scala/gatling-utils/src/test/resources/simulation-log/minimal.txt Normal file
View File

@ -0,0 +1,4 @@
ASSERTION AAIBAAIFAAAAAAAAAAAA
RUN simulation.StandardTwoStepsSimulation standardtwostepssimulation 1587679986754 , args: --ledger abc
USER two-steps.standard 1 START 1587679986860 1587679986860
REQUEST 1 two-steps-sync 1587679987299 1587680007510 OK ACK

4
libs-scala/gatling-utils/src/test/resources/simulation-log/missing-error-message.txt Normal file
View File

@ -0,0 +1,4 @@
ASSERTION AAIBAAIFAAAAAAAAAAAA
RUN simulation.StandardTwoStepsSimulation standardtwostepssimulation 1587679986754 , args: --ledger abc
USER two-steps.standard 1 START 1587679986860 1587679986860
REQUEST 1 two-steps-sync 1587679987299 1587680007510 OK

11
libs-scala/gatling-utils/src/test/resources/simulation-log/multiple-requests.txt Normal file
View File

@ -0,0 +1,11 @@
ASSERTION AAIBAAIFAAAAAAAAAAAA
RUN simulation.StandardTwoStepsSimulation foobar 1587679986754 , args: --ledger abc
USER two-steps.standard 1 START 1587679986860 1587679986860
USER two-steps.standard 2 START 1587679986861 1587679986861
REQUEST 1 sync 1000000000001 1000000000100 OK ACK
REQUEST 1 sync 1000000000002 1000000000100 OK ACK
REQUEST 1 async 1000000000003 1000000000100 OK ACK
REQUEST 1 desync 1000000000004 1000000000100 KO Sum Ting Wong
REQUEST 2 desync 1000000000005 1000000000100 OK Sum Ting Wong
USER two-steps.standard 1 END 1588647388375 1588647409795
USER two-steps.standard 2 END 1588647388376 1588647409795

5
libs-scala/gatling-utils/src/test/resources/simulation-log/multiple-run.txt Normal file
View File

@ -0,0 +1,5 @@
ASSERTION AAIBAAIFAAAAAAAAAAAA
RUN simulation.StandardTwoStepsSimulation standardtwostepssimulation 1587679986754 , args: --ledger abc
RUN simulation.StandardTwoStepsSimulation standardtwostepsslation 1587679986754 , args: --ledger def
USER two-steps.standard 1 START 1587679986860 1587679986860
REQUEST 1 two-steps-sync 1587679987299 1587680007510 OK ACK

7
libs-scala/gatling-utils/src/test/resources/simulation-log/multiple-scenarios-expected-csv.csv Normal file
View File

@ -0,0 +1,7 @@
simulation,scenario,maxUsers,request,start,duration,end,count,successCount,errorCount,min,p90,p95,p99,p999,max,mean,avg,stddev,rps
foo,first,2,async,1000000000003,0.097,1000000000100,1,1,0,97,97,97,97,97,97,97,97,0,10.309278350515465
foo,first,2,desync,1000000000004,0.096,1000000000100,2,1,1,95,95,95,95,95,95,95,95,0,10.416666666666666
foo,first,2,sync,1000000000001,0.099,1000000000100,2,2,0,98,99,99,99,99,99,98,99,1,20.202020202020204
foo,second,3,nosync,2000000000002,0.098,2000000000100,1,1,0,98,98,98,98,98,98,98,98,0,10.204081632653061
foo,second,3,sync,2000000000001,0.1,2000000000101,2,2,0,99,100,100,100,100,100,99,100,1,20.0
foo,third,1,dummy,3000000000002,0.098,3000000000100,1,1,0,98,98,98,98,98,98,98,98,0,10.204081632653061
1 simulation scenario maxUsers request start duration end count successCount errorCount min p90 p95 p99 p999 max mean avg stddev rps
2 foo first 2 async 1000000000003 0.097 1000000000100 1 1 0 97 97 97 97 97 97 97 97 0 10.309278350515465
3 foo first 2 desync 1000000000004 0.096 1000000000100 2 1 1 95 95 95 95 95 95 95 95 0 10.416666666666666
4 foo first 2 sync 1000000000001 0.099 1000000000100 2 2 0 98 99 99 99 99 99 98 99 1 20.202020202020204
5 foo second 3 nosync 2000000000002 0.098 2000000000100 1 1 0 98 98 98 98 98 98 98 98 0 10.204081632653061
6 foo second 3 sync 2000000000001 0.1 2000000000101 2 2 0 99 100 100 100 100 100 99 100 1 20.0
7 foo third 1 dummy 3000000000002 0.098 3000000000100 1 1 0 98 98 98 98 98 98 98 98 0 10.204081632653061

21
libs-scala/gatling-utils/src/test/resources/simulation-log/multiple-scenarios.txt Normal file
View File

@ -0,0 +1,21 @@
ASSERTION AAIBAAIFAAAAAAAAAAAA
RUN simulation.StandardTwoStepsSimulation foo 1587679986754 , args: --ledger abc
USER first 1 START 1587679986860 1587679986860
USER first 2 START 1587679986861 1587679986861
USER second 3 START 1587679986861 1587679986861
USER second 4 START 1587679986861 1587679986861
USER second 5 START 1587679986861 1587679986861
USER third 6 START 1587679986861 1587679986861
REQUEST 1 sync 1000000000001 1000000000100 OK ACK
REQUEST 1 sync 1000000000002 1000000000100 OK ACK
REQUEST 1 async 1000000000003 1000000000100 OK ACK
REQUEST 1 desync 1000000000004 1000000000100 KO Err
REQUEST 2 desync 1000000000005 1000000000100 OK Err
REQUEST 3 sync 2000000000001 2000000000101 OK ACK
REQUEST 3 sync 2000000000002 2000000000101 OK ACK
REQUEST 4 nosync 2000000000002 2000000000100 OK ACK
REQUEST 6 dummy 3000000000002 3000000000100 OK ACK
USER two-steps.standard 1 END 1588647388375 1588647409795
USER two-steps.standard 2 END 1588647388376 1588647409795
USER two-steps.standard 3 END 1588647388376 1588647409795
USER two-steps.standard 4 END 1588647388376 1588647409795

3
libs-scala/gatling-utils/src/test/resources/simulation-log/no-request.txt Normal file
View File

@ -0,0 +1,3 @@
ASSERTION AAIBAAIFAAAAAAAAAAAA
RUN simulation.StandardTwoStepsSimulation standardtwostepssimulation 1587679986754 , args: --ledger abc
USER two-steps.standard 1 START 1587679986860 1587679986860

3
libs-scala/gatling-utils/src/test/resources/simulation-log/no-run.txt Normal file
View File

@ -0,0 +1,3 @@
ASSERTION AAIBAAIFAAAAAAAAAAAA
USER two-steps.standard 1 START 1587679986860 1587679986860
REQUEST 1 two-steps-sync 1587679987299 1587680007510 OK ACK

3
libs-scala/gatling-utils/src/test/resources/simulation-log/no-user.txt Normal file
View File

@ -0,0 +1,3 @@
ASSERTION AAIBAAIFAAAAAAAAAAAA
RUN simulation.StandardTwoStepsSimulation standardtwostepssimulation 1587679986754 , args: --ledger abc
REQUEST 1 two-steps-sync 1587679987299 1587680007510 OK ACK

8
libs-scala/gatling-utils/src/test/resources/simulation-log/with-unknown-entries.txt Normal file
View File

@ -0,0 +1,8 @@
ASSERTION AAIBAAIFAAAAAAAAAAAA
RUN simulation.StandardTwoStepsSimulation standardtwostepssimulation 1587679986754 , args: --ledger abc
USER two-steps.standard 1 START 1587679986860 1587679986860
UNKNOWN foo foobarbar baz
UNKNOWN
REQUEST 1 two-steps-sync 1587679987299 1587680007510 OK ACK
UN-KNOWN foo foobarbar baz
UNKNOWN

45
libs-scala/gatling-utils/src/test/scala/com/daml/gatling/stats/DurationStatisticsSpec.scala Normal file
View File

@ -0,0 +1,45 @@
// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package com.daml.gatling.stats
import com.daml.scalatest.FlatSpecCheckLaws
import org.scalacheck.{Arbitrary, Gen}
import org.scalactic.TypeCheckedTripleEquals
import org.scalatest.prop.GeneratorDrivenPropertyChecks
import org.scalatest.{FlatSpec, Matchers}
import scalaz.Equal
import scalaz.scalacheck.ScalazProperties
class DurationStatisticsSpec
extends FlatSpec
with FlatSpecCheckLaws
with Matchers
with TypeCheckedTripleEquals
with GeneratorDrivenPropertyChecks {
import SimulationLog.DurationStatistics
import DurationStatisticsSpec._
override implicit val generatorDrivenConfig: PropertyCheckConfiguration =
PropertyCheckConfiguration(minSuccessful = 1000)
behavior of s"${classOf[DurationStatistics].getSimpleName} Monoid"
checkLaws(ScalazProperties.monoid.laws[DurationStatistics])
}
private object DurationStatisticsSpec {
import SimulationLog.DurationStatistics
val durationStatisticsGen: Gen[DurationStatistics] = for {
durations <- Gen.listOf(Gen.posNum[Int])
start <- Gen.option(Gen.posNum[Long])
end <- Gen.option(Gen.posNum[Long])
} yield DurationStatistics(durations, start, end)
implicit val durationStatisticsArb: Arbitrary[DurationStatistics] =
Arbitrary(durationStatisticsGen)
implicit val durationStatisticsEqual: Equal[DurationStatistics] = Equal.equalA
}

465
libs-scala/gatling-utils/src/test/scala/com/daml/gatling/stats/SimulationLogSpec.scala Normal file
View File

@ -0,0 +1,465 @@
// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package com.daml.gatling.stats
import java.io.File
import org.scalactic.TypeCheckedTripleEquals
import scalaz.Scalaz._
import scala.util.Random
import com.daml.gatling.stats.util.ReadFileSyntax._
import org.scalatest.{FlatSpec, Matchers}
import com.daml.bazeltools.BazelRunfiles.requiredResource
@SuppressWarnings(Array("org.wartremover.warts.NonUnitStatements"))
class SimulationLogSpec extends FlatSpec with Matchers with TypeCheckedTripleEquals {
import SimulationLog._
behavior of "SimulationLog"
private val simulationLog = "libs-scala/gatling-utils/src/test/resources/simulation-log"
private def resultFor(fileName: String) =
SimulationLog.fromFile(requiredResource(s"$simulationLog/$fileName.txt"))
it should "fail if file does not exist" in {
SimulationLog.fromFile(new File("DOES-NOT-EXIST-OgUzdJsvKHc9TtfNiLXA")) shouldBe 'left
}
it should "fail if no RUN entry" in {
resultFor("no-run") shouldBe 'left
}
it should "fail if no USER entry" in {
resultFor("no-user") shouldBe 'left
}
it should "fail if multiple RUN entries" in {
resultFor("multiple-run") shouldBe 'left
}
it should "return correct result for minimal log" in {
val request = RequestStats(
userId = 1,
requestLabel = "two-steps-sync",
start = 1587679987299L,
end = 1587680007510L,
successful = true
)
val expected = SimulationLog(
simulation = "standardtwostepssimulation",
scenarios = ScenarioStats(
"two-steps.standard",
maxUsers = 1,
requestsByType = Map("two-steps-sync" -> RequestTypeStats.fromRequestStats(request :: Nil))
) :: Nil
)
resultFor("minimal") should ===(expected.right)
}
it should "process requests without status message" in {
val request = RequestStats(
userId = 1,
requestLabel = "two-steps-sync",
start = 1587679987299L,
end = 1587680007510L,
successful = true
)
val expected = SimulationLog(
simulation = "standardtwostepssimulation",
scenarios = ScenarioStats(
label = "two-steps.standard",
maxUsers = 1,
requestsByType = Map("two-steps-sync" -> RequestTypeStats.fromRequestStats(request :: Nil))
) :: Nil
)
resultFor("missing-error-message") should ===(expected.right)
}
it should "group requests by type, with failing requests and multiple users" in {
val expected = SimulationLog(
simulation = "foobar",
scenarios = ScenarioStats(
"two-steps.standard",
maxUsers = 2,
requestsByType = Map(
"sync" -> RequestTypeStats(
DurationStatistics(Seq(99, 98), Some(1000000000001L), Some(1000000000100L)),
mzero[DurationStatistics].copy(start = Some(1000000000001L))),
"desync" -> RequestTypeStats(
DurationStatistics(Seq(95), Some(1000000000004L), Some(1000000000100L)),
DurationStatistics(Seq(96), Some(1000000000004L), Some(1000000000100L))),
"async" -> RequestTypeStats(
DurationStatistics(Seq(97), Some(1000000000003L), Some(1000000000100L)),
mzero[DurationStatistics].copy(start = Some(1000000000003L)))
)
) :: Nil
)
resultFor("multiple-requests") should ===(expected.right)
}
it should "group requests by multiple scenarios and types" in {
val expected = SimulationLog(
simulation = "foo",
scenarios = List(
ScenarioStats(
"third",
maxUsers = 1,
requestsByType = Map(
"dummy" -> RequestTypeStats(
DurationStatistics(Seq(98), Some(3000000000002L), Some(3000000000100L)),
mzero[DurationStatistics].copy(start = Some(3000000000002L))
)
)
),
ScenarioStats(
"second",
maxUsers = 3,
requestsByType = Map(
"sync" -> RequestTypeStats(
DurationStatistics(Seq(100, 99), Some(2000000000001L), Some(2000000000101L)),
mzero[DurationStatistics].copy(start = Some(2000000000001L))
),
"nosync" -> RequestTypeStats(
DurationStatistics(Seq(98), Some(2000000000002L), Some(2000000000100L)),
mzero[DurationStatistics].copy(start = Some(2000000000002L))
)
)
),
ScenarioStats(
"first",
maxUsers = 2,
requestsByType = Map(
"sync" -> RequestTypeStats(
DurationStatistics(Seq(99, 98), Some(1000000000001L), Some(1000000000100L)),
mzero[DurationStatistics].copy(start = Some(1000000000001L))
),
"desync" -> RequestTypeStats(
DurationStatistics(Seq(95), Some(1000000000004L), Some(1000000000100L)),
DurationStatistics(Seq(96), Some(1000000000004L), Some(1000000000100L))
),
"async" -> RequestTypeStats(
DurationStatistics(Seq(97), Some(1000000000003L), Some(1000000000100L)),
mzero[DurationStatistics].copy(start = Some(1000000000003L))
)
)
)
)
)
resultFor("multiple-scenarios") should ===(expected.right)
}
it should "ignore unknown entry types" in {
resultFor("with-unknown-entries") should ===(resultFor("minimal"))
}
it should "produce correct CSV" in {
val expected =
requiredResource(s"$simulationLog/multiple-scenarios-expected-csv.csv").contentsAsString
resultFor("multiple-scenarios")
.map(_.toCsvString)
.getOrElse(throw new AssertionError()) should ===(
expected.getOrElse(throw new AssertionError()))
}
behavior of "DurationStatistics"
it should "calculate correct metrics for empty result" in {
val stats = DurationStatistics(Seq.empty, None, None)
stats should ===(mzero[DurationStatistics])
stats.duration should ===(None)
stats.requestsPerSecond should ===(None)
stats.count should ===(0)
stats.percentile(0.0) should ===(None)
stats.percentile(1.0) should ===(None)
stats.mean should ===(None)
stats.geometricMean should ===(None)
}
it should "calculate correct metrics for start time only" in {
val stats = DurationStatistics(
start = Some(1000L),
end = None,
durations = Seq()
)
stats.duration should ===(None)
stats.requestsPerSecond should ===(None)
stats.count should ===(0)
stats.percentile(0.0) should ===(None)
stats.percentile(1.0) should ===(None)
stats.mean should ===(None)
stats.geometricMean should ===(None)
}
it should "calculate correct metrics for single successful result" in {
val stats = DurationStatistics(
start = Some(5000L),
end = Some(7000L),
durations = Seq(2000)
)
stats.duration should ===(Some(2000))
stats.requestsPerSecond should ===(Some(0.5))
stats.count should ===(1)
stats.percentile(0.0) should ===(Some(2000))
stats.percentile(1.0) should ===(Some(2000))
stats.mean should ===(Some(2000.0))
stats.geometricMean.get should ===(2000.0 +- 0.01)
}
it should "calculate correct metrics for multiple successful results" in {
val stats = DurationStatistics(
start = Some(1000L),
end = Some(5000L),
durations = Seq(2000, 1000, 3000)
)
stats.duration should ===(Some(4000))
stats.requestsPerSecond should ===(Some(3.0 / 4.0))
stats.count should ===(3)
stats.percentile(0.0) should ===(Some(1000))
stats.percentile(0.5) should ===(Some(2000))
stats.percentile(1.0) should ===(Some(3000))
stats.mean should ===(Some(2000.0))
stats.stdDev.get should ===(816.5 +- 0.1)
stats.geometricMean.get should ===(1817.12 +- 0.01)
}
it should "calculate correct metrics for mixed results" in {
val stats = DurationStatistics(
start = Some(1000L),
end = Some(5000L),
durations = Seq(2000, 1000, 3000)
)
stats.duration should ===(Some(4000))
stats.requestsPerSecond should ===(Some(3.0 / 4.0))
stats.percentile(0.0) should ===(Some(1000))
stats.percentile(0.5) should ===(Some(2000))
stats.percentile(1.0) should ===(Some(3000))
stats.stdDev.get should ===(816.5 +- 0.1)
stats.geometricMean.get should ===(1817.12 +- 0.01)
}
it should "calculate correct percentiles for empty result" in {
val stats = mzero[DurationStatistics]
stats.percentile(0.0) should ===(None)
stats.percentile(0.1) should ===(None)
stats.percentile(0.5) should ===(None)
stats.percentile(1.0) should ===(None)
}
it should "calculate correct percentiles and stdDev for single result" in {
val stats = DurationStatistics(
start = Some(1000),
end = Some(4000),
durations = Seq(3000)
)
stats.stdDev.get should ===(0.0)
stats.percentile(0.0).get should ===(3000)
stats.percentile(0.1).get should ===(3000)
stats.percentile(0.5).get should ===(3000)
stats.percentile(1.0).get should ===(3000)
}
it should "calculate correct standard deviation and percentiles" in {
val stats = DurationStatistics(
start = Some(1000),
end = Some(4000),
durations = Seq(100, 500, 1000, 2000, 3000)
)
stats.stdDev.get should ===(1053.38 +- 0.1)
stats.mean.get should ===(1320.0)
stats.percentile(0.0).get should ===(100)
stats.percentile(0.1).get should ===(100)
stats.percentile(0.25).get should ===(500)
stats.percentile(0.5).get should ===(1000)
stats.percentile(0.70).get should ===(2000)
stats.percentile(0.75).get should ===(2000)
stats.percentile(0.8).get should ===(2000)
stats.percentile(0.9).get should ===(3000)
stats.percentile(1.0).get should ===(3000)
}
it should "calculate correct percentiles, mean and stddev for large number of requests" in {
val stats = DurationStatistics(
start = Some(1000),
end = Some(101000),
durations = Random.shuffle(0 to 10000)
)
stats.stdDev.get should ===(2887.04 +- 0.0001)
stats.mean.get should ===(10000.0 / 2)
Range
.BigDecimal(BigDecimal("0.0"), BigDecimal("1.0"), BigDecimal("0.001"))
.foreach(p => stats.percentile(p.toDouble).get should ===((p * 10000).toInt))
}
behavior of "RequestTypeStats"
it should "compose correct stats for empty result" in {
RequestTypeStats.fromRequestStats(Nil) should ===(
RequestTypeStats(
successful = mzero[DurationStatistics],
failed = mzero[DurationStatistics]
))
}
it should "compose correct stats for single failed result" in {
val stats = RequestTypeStats.fromRequestStats(
RequestStats(1, "foo", 1000, 2000, successful = false) :: Nil)
stats should ===(
RequestTypeStats(
successful = mzero[DurationStatistics].copy(start = Some(1000L)),
failed = DurationStatistics(
start = Some(1000L),
end = Some(2000L),
durations = Seq(1000)
)
))
}
it should "compose correct stats for single successful result" in {
val stats = RequestTypeStats.fromRequestStats(
RequestStats(1, "foo", 5000, 7000, successful = true) :: Nil)
stats should ===(
RequestTypeStats(
successful = DurationStatistics(
start = Some(5000L),
end = Some(7000L),
durations = Seq(2000)
),
failed = mzero[DurationStatistics].copy(start = Some(5000L))
)
)
}
it should "compose correct stats for multiple successful results" in {
val stats = RequestTypeStats.fromRequestStats(
List(
RequestStats(1, "foo", 2000, 4000, successful = true),
RequestStats(1, "foo", 1000, 2000, successful = true),
RequestStats(1, "foo", 2000, 5000, successful = true)
)
)
stats should ===(
RequestTypeStats(
successful = DurationStatistics(
start = Some(1000L),
end = Some(5000L),
durations = Seq(2000, 1000, 3000)
),
failed = mzero[DurationStatistics].copy(start = Some(1000L))
)
)
}
it should "compose correct stats for mixed results" in {
val stats = RequestTypeStats.fromRequestStats(
List(
RequestStats(1, "foo", 1000, 4000, successful = false),
RequestStats(1, "foo", 2000, 4000, successful = true),
RequestStats(1, "foo", 1500, 2500, successful = true),
RequestStats(1, "foo", 2000, 5000, successful = true),
RequestStats(1, "foo", 9000, 10000, successful = false)
)
)
stats should ===(
RequestTypeStats(
successful = DurationStatistics(
start = Some(1000L),
end = Some(5000L),
durations = Seq(2000, 1000, 3000)
),
failed = DurationStatistics(
start = Some(1000L),
end = Some(10000L),
durations = Seq(3000, 1000)
)
)
)
}
it should "be monoidal" in {
val stats1 = RequestStats(1, "foo", 1000, 2000, successful = true)
val stats2 = RequestStats(1, "foo", 2000, 4000, successful = true)
val stats3 = RequestStats(1, "foo", 2000, 5000, successful = false)
val stats4 = RequestStats(1, "foo", 5000, 5500, successful = true)
RequestTypeStats.fromRequestStats(Nil) |+| RequestTypeStats.fromRequestStats(Nil) should be(
RequestTypeStats.fromRequestStats(Nil))
RequestTypeStats.fromRequestStats(Nil) |+| RequestTypeStats.fromRequestStats(stats1 :: Nil) should be(
RequestTypeStats.fromRequestStats(stats1 :: Nil))
RequestTypeStats.fromRequestStats(stats1 :: Nil) |+| RequestTypeStats.fromRequestStats(Nil) should be(
RequestTypeStats.fromRequestStats(stats1 :: Nil))
RequestTypeStats.fromRequestStats(stats1 :: Nil) |+| RequestTypeStats.fromRequestStats(
stats2 :: Nil) should be(RequestTypeStats.fromRequestStats(stats1 :: stats2 :: Nil))
RequestTypeStats.fromRequestStats(stats3 :: Nil) |+| RequestTypeStats.fromRequestStats(
stats2 :: Nil) should be(RequestTypeStats.fromRequestStats(stats3 :: stats2 :: Nil))
RequestTypeStats.fromRequestStats(stats1 :: stats2 :: Nil) |+| RequestTypeStats
.fromRequestStats(stats3 :: stats4 :: Nil) should be {
RequestTypeStats.fromRequestStats(stats1 :: stats2 :: stats3 :: stats4 :: Nil)
}
}
it should "render correctly when populated" in {
RequestTypeStats(
DurationStatistics(
start = Some(1000),
end = Some(101000),
durations = Random.shuffle((1000 until 191000 by 100).toVector)
),
DurationStatistics(
start = Some(1000),
end = Some(201000),
durations = Random.shuffle((2000 until 2100).toVector)
)
).formatted("foobar") should ===(
"""================================================================================
|---- foobar --------------------------------------------------------------------
|> Number of requests 2000 (OK=1900 KO=100 )
|> Min. response time 1000 (OK=1000 KO=2000 )
|> Max. response time 190900 (OK=190900 KO=2099 )
|> Mean response time 91255 (OK=95950 KO=2050 )
|> Std. deviation 57243 (OK=54848 KO=29 )
|> response time 90th percentile 170900 (OK=171900 KO=2089 )
|> response time 95th percentile 180900 (OK=181400 KO=2094 )
|> response time 99th percentile 188900 (OK=189000 KO=2098 )
|> response time 99.9th percentile 190700 (OK=190700 KO=2099 )
|> Mean requests/second 10 (OK=19 KO=0.5 )
|---- Response time distribution ------------------------------------------------
|> t < 5000 ms 40 ( 2%)
|> 5000 ms < t < 30000 ms 250 (12.5%)
|> 30000 ms < t 1610 (80.5%)
|> failed 100 ( 5%)
|================================================================================""".stripMargin)
}
it should "render correctly when not populated" in {
RequestTypeStats(mzero[DurationStatistics], mzero[DurationStatistics])
.formatted("foobarbaz") should ===(
"""================================================================================
|---- foobarbaz -----------------------------------------------------------------
|> Number of requests - (OK=- KO=- )
|> Min. response time - (OK=- KO=- )
|> Max. response time - (OK=- KO=- )
|> Mean response time - (OK=- KO=- )
|> Std. deviation - (OK=- KO=- )
|> response time 90th percentile - (OK=- KO=- )
|> response time 95th percentile - (OK=- KO=- )
|> response time 99th percentile - (OK=- KO=- )
|> response time 99.9th percentile - (OK=- KO=- )
|> Mean requests/second - (OK=- KO=- )
|---- Response time distribution ------------------------------------------------
|> t < 5000 ms 0 ( -%)
|> 5000 ms < t < 30000 ms 0 ( -%)
|> 30000 ms < t 0 ( -%)
|> failed 0 ( -%)
|================================================================================""".stripMargin)
}
}