[User management] Add race conditions test-tool tests for CreateUser and GrantUserRights [DPP-847] (#12983)

changelog_begin
changelog_end

ledger/error/src/main/scala/com/daml/error/definitions/IndexErrors.scala

@@ -84,4 +84,5 @@ object IndexErrors extends IndexErrorGroup {
 
   case class IndexDbException(status: io.grpc.Status, metadata: io.grpc.Metadata)
       extends LoggingApiException(status, metadata)
+
 }

ledger/error/src/main/scala/com/daml/error/utils/ErrorDetails.scala

@@ -3,21 +3,28 @@
 
 package com.daml.error.utils
 
+import com.daml.error.{BaseError, ErrorCode}
 import com.google.protobuf
 import com.google.rpc.{ErrorInfo, RequestInfo, ResourceInfo, RetryInfo}
+import io.grpc.{Status, StatusRuntimeException}
+import io.grpc.protobuf.StatusProto
 
 import scala.jdk.CollectionConverters._
 import scala.concurrent.duration._
 
 object ErrorDetails {
+
   sealed trait ErrorDetail extends Product with Serializable
 
   final case class ResourceInfoDetail(name: String, typ: String) extends ErrorDetail
-  final case class ErrorInfoDetail(reason: String, metadata: Map[String, String])
+  final case class ErrorInfoDetail(errorCodeId: String, metadata: Map[String, String])
       extends ErrorDetail
   final case class RetryInfoDetail(duration: Duration) extends ErrorDetail
   final case class RequestInfoDetail(requestId: String) extends ErrorDetail
 
+  def from(e: StatusRuntimeException): Seq[ErrorDetail] =
+    from(StatusProto.fromThrowable(e).getDetailsList.asScala.toSeq)
+
   def from(anys: Seq[protobuf.Any]): Seq[ErrorDetail] = anys.toList.map {
     case any if any.is(classOf[ResourceInfo]) =>
       val v = any.unpack(classOf[ResourceInfo])
@@ -39,4 +46,36 @@ object ErrorDetails {
 
     case any => throw new IllegalStateException(s"Could not unpack value of: |$any|")
   }
+
+  def isInternalError(t: Throwable): Boolean = t match {
+    case e: StatusRuntimeException => isInternalError(e)
+    case _ => false
+  }
+
+  def isInternalError(e: StatusRuntimeException): Boolean =
+    e.getStatus.getCode == Status.Code.INTERNAL && e.getStatus.getDescription.startsWith(
+      BaseError.SecuritySensitiveMessageOnApiPrefix
+    )
+
+  /** @return whether a status runtime exception matches the error code.
+    *
+    * NOTE: This method is not suitable for:
+    * 1) security sensitive error codes (e.g. internal or authentication related) as they are stripped from all the details when being converted to instances of [[StatusRuntimeException]],
+    * 2) error codes that do not translate to gRPC level errors (i.e. error codes that don't have a corresponding gRPC status)
+    */
+  def matches(e: StatusRuntimeException, errorCode: ErrorCode): Boolean = {
+    val matchesErrorCodeId = from(e).exists {
+      case ErrorInfoDetail(errorCodeId, _) => errorCodeId == errorCode.id
+      case _ => false
+    }
+    val matchesMessagePrefix = e.getStatus.getDescription.startsWith(errorCode.id)
+    val matchesStatusCode = errorCode.category.grpcCode.contains(e.getStatus.getCode)
+    matchesErrorCodeId && matchesMessagePrefix && matchesStatusCode
+  }
+
+  def matches(t: Throwable, errorCode: ErrorCode): Boolean = t match {
+    case e: StatusRuntimeException => matches(e, errorCode)
+    case _ => false
+  }
+
 }

ledger/ledger-api-tests/infrastructure/src/main/scala/com/daml/ledger/api/testtool/infrastructure/Assertions.scala

@@ -173,7 +173,7 @@ object Assertions {
     val actualStatusCode = status.getCode
     val actualErrorDetails = ErrorDetails.from(status.getDetailsList.asScala.toSeq)
     val actualErrorId = actualErrorDetails
-      .collectFirst { case err: ErrorDetails.ErrorInfoDetail => err.reason }
+      .collectFirst { case err: ErrorDetails.ErrorInfoDetail => err.errorCodeId }
       .getOrElse(fail("Actual error id is not defined"))
     val actualRetryability = actualErrorDetails
       .collectFirst { case err: ErrorDetails.RetryInfoDetail => err.duration }

ledger/ledger-api-tests/suites/src/main/scala/com/daml/ledger/api/testtool/suites/v1_8/UserManagementServiceIT.scala

@@ -6,7 +6,8 @@ package com.daml.ledger.api.testtool.suites.v1_8
 import java.util.UUID
 
 import com.daml.error.ErrorCode
-import com.daml.error.definitions.LedgerApiErrors
+import com.daml.error.definitions.{IndexErrors, LedgerApiErrors}
+import com.daml.error.utils.ErrorDetails
 import com.daml.ledger.api.testtool.infrastructure.Allocation._
 import com.daml.ledger.api.testtool.infrastructure.Assertions._
 import com.daml.ledger.api.testtool.infrastructure.LedgerTestSuite
@@ -31,6 +32,7 @@ import com.daml.ledger.api.v1.admin.user_management_service.{
 import com.daml.ledger.api.v1.admin.{user_management_service => proto}
 
 import scala.concurrent.{ExecutionContext, Future}
+import scala.util.Success
 
 final class UserManagementServiceIT extends LedgerTestSuite {
 
@@ -178,6 +180,73 @@ final class UserManagementServiceIT extends LedgerTestSuite {
     } yield ()
   })
 
+  userManagementTest(
+    "CreateUsersRaceCondition",
+    "Tests scenario of multiple concurrent create-user calls for the same user",
+    runConcurrently = false,
+  ) {
+    implicit ec =>
+      { participant =>
+        val attempts = (1 to 10).toVector
+        val userId = participant.nextUserId()
+        val request =
+          CreateUserRequest(Some(User(id = userId, primaryParty = "")), rights = Seq.empty)
+        for {
+          results <- Future
+            .traverse(attempts) { _ =>
+              participant.createUser(request).transform(Success(_))
+            }
+        } yield {
+          assertSingleton(
+            "successful user creation",
+            results.filter(_.isSuccess),
+          )
+          val unexpectedErrors = results
+            .collect { case x if x.isFailure => x.failed.get }
+            .filterNot(t =>
+              ErrorDetails.matches(t, LedgerApiErrors.AdminServices.UserAlreadyExists) ||
+                ErrorDetails.matches(t, IndexErrors.DatabaseErrors.SqlTransientError) ||
+                ErrorDetails.isInternalError(t)
+            )
+          assertSameElements(actual = unexpectedErrors, expected = Seq.empty)
+        }
+      }
+  }
+
+  userManagementTest(
+    "GrantRightsRaceCondition",
+    "Tests scenario of multiple concurrent grant-right calls for the same user and the same rights",
+    runConcurrently = false,
+  ) {
+    implicit ec =>
+      { participant =>
+        val attempts = (1 to 10).toVector
+        val userId = participant.nextUserId()
+        val createUserRequest =
+          CreateUserRequest(Some(User(id = userId, primaryParty = "")), rights = Seq.empty)
+        val grantRightsRequest = GrantUserRightsRequest(userId = userId, rights = userRightsBatch)
+        for {
+          _ <- participant.createUser(createUserRequest)
+          results <- Future.traverse(attempts) { _ =>
+            participant.userManagement.grantUserRights(grantRightsRequest).transform(Success(_))
+          }
+        } yield {
+          assert(
+            results.exists(_.isSuccess),
+            "Expected at least one successful user right grant",
+          )
+          val unexpectedErrors = results
+            .collect { case x if x.isFailure => x.failed.get }
+            // Note: `IndexErrors.DatabaseErrors.SqlNonTransientError` is signalled on H2 and the original cause being `org.h2.jdbc.JdbcSQLIntegrityConstraintViolationException`
+            .filterNot(e =>
+              ErrorDetails.isInternalError(e) || ErrorDetails
+                .matches(e, IndexErrors.DatabaseErrors.SqlTransientError)
+            )
+          assertSameElements(actual = unexpectedErrors, expected = Seq.empty)
+        }
+      }
+  }
+
   userManagementTest(
     "TestAdminExists",
     "Ensure admin user exists",

ledger/sandbox-on-x/src/test/suite/scala/com/daml/ledger/sandbox/bridge/validate/SequenceSpec.scala

@@ -588,7 +588,7 @@ class SequenceSpec
     val actualStatusCode = status.getCode
     val actualErrorDetails = ErrorDetails.from(status.getDetailsList.asScala.toSeq)
     val actualErrorId = actualErrorDetails
-      .collectFirst { case err: ErrorDetails.ErrorInfoDetail => err.reason }
+      .collectFirst { case err: ErrorDetails.ErrorInfoDetail => err.errorCodeId }
       .getOrElse(fail("Actual error id is not defined"))
     val actualRetryability = actualErrorDetails
       .collectFirst { case err: ErrorDetails.RetryInfoDetail => err.duration }