diff --git a/ledger/error/src/main/scala/com/daml/error/definitions/IndexErrors.scala b/ledger/error/src/main/scala/com/daml/error/definitions/IndexErrors.scala index d18b0be180..63ed29cba1 100644 --- a/ledger/error/src/main/scala/com/daml/error/definitions/IndexErrors.scala +++ b/ledger/error/src/main/scala/com/daml/error/definitions/IndexErrors.scala @@ -84,4 +84,5 @@ object IndexErrors extends IndexErrorGroup { case class IndexDbException(status: io.grpc.Status, metadata: io.grpc.Metadata) extends LoggingApiException(status, metadata) + } diff --git a/ledger/error/src/main/scala/com/daml/error/utils/ErrorDetails.scala b/ledger/error/src/main/scala/com/daml/error/utils/ErrorDetails.scala index db3d34597c..67385cf08d 100644 --- a/ledger/error/src/main/scala/com/daml/error/utils/ErrorDetails.scala +++ b/ledger/error/src/main/scala/com/daml/error/utils/ErrorDetails.scala @@ -3,21 +3,28 @@ package com.daml.error.utils +import com.daml.error.{BaseError, ErrorCode} import com.google.protobuf import com.google.rpc.{ErrorInfo, RequestInfo, ResourceInfo, RetryInfo} +import io.grpc.{Status, StatusRuntimeException} +import io.grpc.protobuf.StatusProto import scala.jdk.CollectionConverters._ import scala.concurrent.duration._ object ErrorDetails { + sealed trait ErrorDetail extends Product with Serializable final case class ResourceInfoDetail(name: String, typ: String) extends ErrorDetail - final case class ErrorInfoDetail(reason: String, metadata: Map[String, String]) + final case class ErrorInfoDetail(errorCodeId: String, metadata: Map[String, String]) extends ErrorDetail final case class RetryInfoDetail(duration: Duration) extends ErrorDetail final case class RequestInfoDetail(requestId: String) extends ErrorDetail + def from(e: StatusRuntimeException): Seq[ErrorDetail] = + from(StatusProto.fromThrowable(e).getDetailsList.asScala.toSeq) + def from(anys: Seq[protobuf.Any]): Seq[ErrorDetail] = anys.toList.map { case any if any.is(classOf[ResourceInfo]) => val v = any.unpack(classOf[ResourceInfo]) @@ -39,4 +46,36 @@ object ErrorDetails { case any => throw new IllegalStateException(s"Could not unpack value of: |$any|") } + + def isInternalError(t: Throwable): Boolean = t match { + case e: StatusRuntimeException => isInternalError(e) + case _ => false + } + + def isInternalError(e: StatusRuntimeException): Boolean = + e.getStatus.getCode == Status.Code.INTERNAL && e.getStatus.getDescription.startsWith( + BaseError.SecuritySensitiveMessageOnApiPrefix + ) + + /** @return whether a status runtime exception matches the error code. + * + * NOTE: This method is not suitable for: + * 1) security sensitive error codes (e.g. internal or authentication related) as they are stripped from all the details when being converted to instances of [[StatusRuntimeException]], + * 2) error codes that do not translate to gRPC level errors (i.e. error codes that don't have a corresponding gRPC status) + */ + def matches(e: StatusRuntimeException, errorCode: ErrorCode): Boolean = { + val matchesErrorCodeId = from(e).exists { + case ErrorInfoDetail(errorCodeId, _) => errorCodeId == errorCode.id + case _ => false + } + val matchesMessagePrefix = e.getStatus.getDescription.startsWith(errorCode.id) + val matchesStatusCode = errorCode.category.grpcCode.contains(e.getStatus.getCode) + matchesErrorCodeId && matchesMessagePrefix && matchesStatusCode + } + + def matches(t: Throwable, errorCode: ErrorCode): Boolean = t match { + case e: StatusRuntimeException => matches(e, errorCode) + case _ => false + } + } diff --git a/ledger/ledger-api-tests/infrastructure/src/main/scala/com/daml/ledger/api/testtool/infrastructure/Assertions.scala b/ledger/ledger-api-tests/infrastructure/src/main/scala/com/daml/ledger/api/testtool/infrastructure/Assertions.scala index c7c39dc557..1c47559725 100644 --- a/ledger/ledger-api-tests/infrastructure/src/main/scala/com/daml/ledger/api/testtool/infrastructure/Assertions.scala +++ b/ledger/ledger-api-tests/infrastructure/src/main/scala/com/daml/ledger/api/testtool/infrastructure/Assertions.scala @@ -173,7 +173,7 @@ object Assertions { val actualStatusCode = status.getCode val actualErrorDetails = ErrorDetails.from(status.getDetailsList.asScala.toSeq) val actualErrorId = actualErrorDetails - .collectFirst { case err: ErrorDetails.ErrorInfoDetail => err.reason } + .collectFirst { case err: ErrorDetails.ErrorInfoDetail => err.errorCodeId } .getOrElse(fail("Actual error id is not defined")) val actualRetryability = actualErrorDetails .collectFirst { case err: ErrorDetails.RetryInfoDetail => err.duration } diff --git a/ledger/ledger-api-tests/suites/src/main/scala/com/daml/ledger/api/testtool/suites/v1_8/UserManagementServiceIT.scala b/ledger/ledger-api-tests/suites/src/main/scala/com/daml/ledger/api/testtool/suites/v1_8/UserManagementServiceIT.scala index 564dafa689..a568addd11 100644 --- a/ledger/ledger-api-tests/suites/src/main/scala/com/daml/ledger/api/testtool/suites/v1_8/UserManagementServiceIT.scala +++ b/ledger/ledger-api-tests/suites/src/main/scala/com/daml/ledger/api/testtool/suites/v1_8/UserManagementServiceIT.scala @@ -6,7 +6,8 @@ package com.daml.ledger.api.testtool.suites.v1_8 import java.util.UUID import com.daml.error.ErrorCode -import com.daml.error.definitions.LedgerApiErrors +import com.daml.error.definitions.{IndexErrors, LedgerApiErrors} +import com.daml.error.utils.ErrorDetails import com.daml.ledger.api.testtool.infrastructure.Allocation._ import com.daml.ledger.api.testtool.infrastructure.Assertions._ import com.daml.ledger.api.testtool.infrastructure.LedgerTestSuite @@ -31,6 +32,7 @@ import com.daml.ledger.api.v1.admin.user_management_service.{ import com.daml.ledger.api.v1.admin.{user_management_service => proto} import scala.concurrent.{ExecutionContext, Future} +import scala.util.Success final class UserManagementServiceIT extends LedgerTestSuite { @@ -178,6 +180,73 @@ final class UserManagementServiceIT extends LedgerTestSuite { } yield () }) + userManagementTest( + "CreateUsersRaceCondition", + "Tests scenario of multiple concurrent create-user calls for the same user", + runConcurrently = false, + ) { + implicit ec => + { participant => + val attempts = (1 to 10).toVector + val userId = participant.nextUserId() + val request = + CreateUserRequest(Some(User(id = userId, primaryParty = "")), rights = Seq.empty) + for { + results <- Future + .traverse(attempts) { _ => + participant.createUser(request).transform(Success(_)) + } + } yield { + assertSingleton( + "successful user creation", + results.filter(_.isSuccess), + ) + val unexpectedErrors = results + .collect { case x if x.isFailure => x.failed.get } + .filterNot(t => + ErrorDetails.matches(t, LedgerApiErrors.AdminServices.UserAlreadyExists) || + ErrorDetails.matches(t, IndexErrors.DatabaseErrors.SqlTransientError) || + ErrorDetails.isInternalError(t) + ) + assertSameElements(actual = unexpectedErrors, expected = Seq.empty) + } + } + } + + userManagementTest( + "GrantRightsRaceCondition", + "Tests scenario of multiple concurrent grant-right calls for the same user and the same rights", + runConcurrently = false, + ) { + implicit ec => + { participant => + val attempts = (1 to 10).toVector + val userId = participant.nextUserId() + val createUserRequest = + CreateUserRequest(Some(User(id = userId, primaryParty = "")), rights = Seq.empty) + val grantRightsRequest = GrantUserRightsRequest(userId = userId, rights = userRightsBatch) + for { + _ <- participant.createUser(createUserRequest) + results <- Future.traverse(attempts) { _ => + participant.userManagement.grantUserRights(grantRightsRequest).transform(Success(_)) + } + } yield { + assert( + results.exists(_.isSuccess), + "Expected at least one successful user right grant", + ) + val unexpectedErrors = results + .collect { case x if x.isFailure => x.failed.get } + // Note: `IndexErrors.DatabaseErrors.SqlNonTransientError` is signalled on H2 and the original cause being `org.h2.jdbc.JdbcSQLIntegrityConstraintViolationException` + .filterNot(e => + ErrorDetails.isInternalError(e) || ErrorDetails + .matches(e, IndexErrors.DatabaseErrors.SqlTransientError) + ) + assertSameElements(actual = unexpectedErrors, expected = Seq.empty) + } + } + } + userManagementTest( "TestAdminExists", "Ensure admin user exists", diff --git a/ledger/sandbox-on-x/src/test/suite/scala/com/daml/ledger/sandbox/bridge/validate/SequenceSpec.scala b/ledger/sandbox-on-x/src/test/suite/scala/com/daml/ledger/sandbox/bridge/validate/SequenceSpec.scala index ef4140b499..246f8186d3 100644 --- a/ledger/sandbox-on-x/src/test/suite/scala/com/daml/ledger/sandbox/bridge/validate/SequenceSpec.scala +++ b/ledger/sandbox-on-x/src/test/suite/scala/com/daml/ledger/sandbox/bridge/validate/SequenceSpec.scala @@ -588,7 +588,7 @@ class SequenceSpec val actualStatusCode = status.getCode val actualErrorDetails = ErrorDetails.from(status.getDetailsList.asScala.toSeq) val actualErrorId = actualErrorDetails - .collectFirst { case err: ErrorDetails.ErrorInfoDetail => err.reason } + .collectFirst { case err: ErrorDetails.ErrorInfoDetail => err.errorCodeId } .getOrElse(fail("Actual error id is not defined")) val actualRetryability = actualErrorDetails .collectFirst { case err: ErrorDetails.RetryInfoDetail => err.duration }