update canton to 20231124.11584.0.v33f28bad/2.8.0-snapshot.20231124.11584.0.v33f28bad (#17911)

CHANGELOG_BEGIN
CHANGELOG_END

Co-authored-by: Azure Pipelines Daml Build <support@digitalasset.com>
This commit is contained in:
azure-pipelines[bot] 2023-11-27 08:20:40 +01:00 committed by GitHub
parent 5775dc1fd5
commit 9c85632f62
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 203 additions and 155 deletions

View File

@ -1,4 +1,4 @@
sdk-version: 2.8.0-snapshot.20231118.12382.0.v86cb8054
sdk-version: 2.8.0-snapshot.20231121.12391.0.v6eaf5081
sandbox-options:
- --wall-clock-time
name: contact

View File

@ -1,4 +1,4 @@
sdk-version: 2.8.0-snapshot.20231118.12382.0.v86cb8054
sdk-version: 2.8.0-snapshot.20231121.12391.0.v6eaf5081
sandbox-options:
- --wall-clock-time
name: message

View File

@ -55,6 +55,7 @@ final case class DomainTopologyTransactionMessage private (
domainId,
notSequencedAfter,
hashOps,
representativeProtocolVersion,
)
private[messages] def toProtoV0: v0.DomainTopologyTransactionMessage =
@ -127,7 +128,15 @@ object DomainTopologyTransactionMessage
ProtoVersion(1) -> VersionedProtoConverter(ProtocolVersion.v5)(
v1.DomainTopologyTransactionMessage
)(
supportedProtoVersion(_)(fromProtoV1),
supportedProtoVersion(_)(fromProtoV1(ProtoVersion(1))),
_.toProtoV1.toByteString,
),
// use separate ProtoVersion starting with v6 to allow different hashing
// scheme for protocol version 5 and 6
ProtoVersion(2) -> VersionedProtoConverter(ProtocolVersion.v6)(
v1.DomainTopologyTransactionMessage
)(
supportedProtoVersion(_)(fromProtoV1(ProtoVersion(2))),
_.toProtoV1.toByteString,
),
)
@ -144,17 +153,38 @@ object DomainTopologyTransactionMessage
domainId: DomainId,
notSequencedAfter: Option[CantonTimestamp],
hashOps: HashOps,
representativeProtocolVersion: RepresentativeProtocolVersion[
DomainTopologyTransactionMessage.type
],
): Hash = {
val builder = hashOps
.build(HashPurpose.DomainTopologyTransactionMessageSignature)
.add(domainId.toProtoPrimitive)
notSequencedAfter.foreach { ts =>
builder.add(ts.toEpochMilli)
val version = representativeProtocolVersion.representative
notSequencedAfter match {
case Some(ts) if version == ProtocolVersion.v5 =>
builder.add(ts.toEpochMilli).discard
case Some(ts) if version >= ProtocolVersion.v6 =>
builder
.add(version.toProtoPrimitive)
.add(ts.toMicros)
.add(transactions.length)
.discard
case Some(_) =>
throw new IllegalStateException(
"notSequencedAfter not expected for pv < 5"
)
case None if version >= ProtocolVersion.v5 =>
// Won't happen because of the invariant
throw new IllegalStateException(
"notSequencedAfter must be present for protocol version >= 5"
)
case None =>
}
transactions.foreach(elem => builder.add(elem.getCryptographicEvidence))
builder.finish()
}
def create(
@ -170,14 +200,16 @@ object DomainTopologyTransactionMessage
val notSequencedAfterUpdated =
notSequencedAfterInvariant.orValue(notSequencedAfter, protocolVersion)
val representativeProtocolVersion = protocolVersionRepresentativeFor(protocolVersion)
val hashToSign = hash(
transactions,
domainId,
notSequencedAfterUpdated,
syncCrypto.crypto.pureCrypto,
representativeProtocolVersion,
)
for {
signature <- syncCrypto.sign(hashToSign).leftMap(_.toString)
domainTopologyTransactionMessageE = Either
.catchOnly[IllegalArgumentException](
@ -186,7 +218,7 @@ object DomainTopologyTransactionMessage
transactions,
notSequencedAfter = notSequencedAfterUpdated,
domainId,
)(protocolVersionRepresentativeFor(protocolVersion))
)(representativeProtocolVersion)
)
.leftMap(_.getMessage)
domainTopologyTransactionMessage <- EitherT.fromEither[Future](
@ -233,7 +265,7 @@ object DomainTopologyTransactionMessage
)(protocolVersionRepresentativeFor(ProtoVersion(0)))
}
private[messages] def fromProtoV1(
private[messages] def fromProtoV1(protoVersion: ProtoVersion)(
message: v1.DomainTopologyTransactionMessage
): ParsingResult[DomainTopologyTransactionMessage] = {
val v1.DomainTopologyTransactionMessage(signature, domainId, timestamp, transactions) = message
@ -253,7 +285,7 @@ object DomainTopologyTransactionMessage
succeededContent,
notSequencedAfter = Some(notSequencedAfter),
DomainId(domainUid),
)(protocolVersionRepresentativeFor(ProtoVersion(1)))
)(protocolVersionRepresentativeFor(protoVersion))
}
override def name: String = "DomainTopologyTransactionMessage"

View File

@ -204,7 +204,7 @@ object EnvelopeContent extends HasProtocolVersionedWithContextCompanion[Envelope
case Content.InformeeMessage(messageP) =>
InformeeMessage.fromProtoV1(hashOps)(messageP)
case Content.DomainTopologyTransactionMessage(messageP) =>
DomainTopologyTransactionMessage.fromProtoV1(messageP)
DomainTopologyTransactionMessage.fromProtoV1(ProtoVersion(1))(messageP)
case Content.EncryptedViewMessage(messageP) =>
EncryptedViewMessageV1.fromProto(messageP)
case Content.SignedMessage(messageP) =>
@ -233,7 +233,7 @@ object EnvelopeContent extends HasProtocolVersionedWithContextCompanion[Envelope
case Content.InformeeMessage(messageP) =>
InformeeMessage.fromProtoV1(hashOps)(messageP)
case Content.DomainTopologyTransactionMessage(messageP) =>
DomainTopologyTransactionMessage.fromProtoV1(messageP)
DomainTopologyTransactionMessage.fromProtoV1(ProtoVersion(2))(messageP)
case Content.EncryptedViewMessage(messageP) =>
EncryptedViewMessageV2.fromProto(messageP)
case Content.SignedMessage(messageP) =>
@ -263,7 +263,7 @@ object EnvelopeContent extends HasProtocolVersionedWithContextCompanion[Envelope
case Content.InformeeMessage(messageP) =>
InformeeMessage.fromProtoV1(hashOps)(messageP)
case Content.DomainTopologyTransactionMessage(messageP) =>
DomainTopologyTransactionMessage.fromProtoV1(messageP)
DomainTopologyTransactionMessage.fromProtoV1(ProtoVersion(2))(messageP)
case Content.EncryptedViewMessage(messageP) =>
EncryptedViewMessageV2.fromProto(messageP)
case Content.TransferOutMediatorMessage(messageP) =>

View File

@ -212,7 +212,7 @@ abstract class RetryWithDelay(
case outcome =>
// this will also log the exception in outcome
val errorKind = retryable.retryOK(outcome, logger)
val errorKind = retryable.retryOK(outcome, logger, Some(lastErrorKind))
val retriesOfErrorKind = if (errorKind == lastErrorKind) retriesOfLastErrorKind else 0
if (
errorKind.maxRetries == Int.MaxValue || retriesOfErrorKind < errorKind.maxRetries
@ -229,23 +229,20 @@ abstract class RetryWithDelay(
directExecutionContext
)
} else {
if (errorKind == lastErrorKind) {
logger.trace(s"Kind of error has not changed since last attempt: $errorKind")
} else {
logger.info(messageOfOutcome(outcome, s"New kind of error: $errorKind."))
// No need to log the exception in outcome, as this has been logged by retryable.retryOk.
}
val level = retryLogLevel.getOrElse {
if (totalRetries < complainAfterRetries || totalMaxRetries != Int.MaxValue)
Level.INFO
else Level.WARN
}
val change = if (errorKind == lastErrorKind) {
""
} else {
s"New kind of error: $errorKind. "
}
LoggerUtil.logAtLevel(
level,
messageOfOutcome(outcome, show"Retrying after $delay."),
// No need to log the exception in outcome, as this has been logged by retryable.retryOk.
messageOfOutcome(outcome, show"${change}Retrying after $delay."),
// No need to log the exception in the outcome, as this has been logged by retryable.retryOk.
)
val delayedF =
@ -512,7 +509,7 @@ final case class When(
else depends(res)(task, retryable)
}(directExecutionContext)
.recoverWith { case NonFatal(e) =>
if (depends.isDefinedAt(e) && retryable.retryOK(Failure(e), logger).maxRetries > 0)
if (depends.isDefinedAt(e) && retryable.retryOK(Failure(e), logger, None).maxRetries > 0)
depends(e)(task, retryable)
else fut
}(directExecutionContext)

View File

@ -26,7 +26,7 @@ object RetryUtil {
*
* Also logs the embedded exception.
*/
def retryOK(outcome: Try[_], logger: TracedLogger)(implicit
def retryOK(outcome: Try[_], logger: TracedLogger, lastErrorKind: Option[ErrorKind])(implicit
tc: TraceContext
): ErrorKind
@ -90,124 +90,144 @@ object RetryUtil {
): Boolean = {
// Don't retry forever on "contention" errors, as these may not actually be due to contention and get stuck
// forever. Eg unique constraint violation exceptions can be caused by contention in H2 leading to data anomalies.
DbExceptionRetryable.retryOK(Failure(error), logger).maxRetries == Int.MaxValue
DbExceptionRetryable.retryOK(Failure(error), logger, None).maxRetries == Int.MaxValue
}
@tailrec override def retryOK(outcome: Try[_], logger: TracedLogger)(implicit
override def retryOK(
outcome: Try[_],
logger: TracedLogger,
lastErrorKind: Option[ErrorKind],
)(implicit
tc: TraceContext
): ErrorKind = {
outcome match {
case util.Success(_) => NoErrorKind
case Failure(exception) =>
logThrowable(exception, logger)
exception match {
case exn: java.util.concurrent.RejectedExecutionException =>
// This occurs when slick's task queue is full
// Create a CantonError so that the error code gets logged.
DatabaseTaskRejected(exn.toString)(
ErrorLoggingContext.fromTracedLogger(logger)
).discard
TransientErrorKind
case exception: PSQLException =>
// Error codes documented here: https://www.postgresql.org/docs/9.6/errcodes-appendix.html
val error = exception.getSQLState
if (error.startsWith("08")) {
// Class 08 Connection Exception
TransientErrorKind
} else if (error == "40001") {
// Class 40 Transaction Rollback: 40001 serialization_failure
// Failure to serialize db accesses, happens due to contention
TransientErrorKind
} else if (error == "40P01") {
// Deadlock
// See DatabaseDeadlockTestPostgres
// This also covers deadlocks reported as BatchUpdateExceptions,
// because they refer to a PSQLException has cause.
TransientErrorKind
} else if (error == "25006") {
// Retry on read only transaction, which can occur on Azure
TransientErrorKind
} else if (error.startsWith("57P") && error != "57P014" && error != "57P04") {
// Retry on operator invention errors, otherwise Canton components crash in an uncontrolled manner when
// the exception bubbles up (don't retry on `query_canceled` and `database_dropped`)
TransientErrorKind
} else {
// Don't retry on other exceptions. These other exceptions should be those for which retrying typically won't
// help, for example a unique constraint violation.
FatalErrorKind
}
case _: SQLIntegrityConstraintViolationException =>
// Both H2 and Oracle may fail with spurious constraint violations, due to racy implementation of the MERGE statements.
// In H2, this may also occur because it does not properly implement the serializable isolation level.
// See UpsertTestOracle
// See https://github.com/h2database/h2database/issues/2167
SpuriousTransientErrorKind
case _: SQLRecoverableException | _: SQLTransientException |
_: SQLNonTransientConnectionException =>
TransientErrorKind
case ex: SQLException =>
val code = ex.getErrorCode
if (ex.getErrorCode == 1) {
// Retry on ORA-00001: unique constraint violated exception
SpuriousTransientErrorKind
} else if (ex.getMessage == "Connection is closed") {
// May fail with a "Connection is closed" message if the db has gone down
TransientErrorKind
} else if (ex.getErrorCode == 4021) {
// ORA timeout occurred while waiting to lock object
TransientErrorKind
} else if (ex.getErrorCode == 54) {
// ORA timeout occurred while waiting to lock object or because NOWAIT has been set
// e.g. as part of truncate table
TransientErrorKind
} else if (ex.getErrorCode == 60) {
// Deadlock
// See DatabaseDeadlockTestOracle
TransientErrorKind
} else if (
ex.getErrorCode == 604 &&
List("ORA-08176", "ORA-08177").exists(ex.getMessage.contains)
) {
// Oracle failure in a batch operation
// For Oracle, the `cause` is not always set properly for exceptions. This is a problem for batched queries.
// So, look through an exception's `message` to see if it contains a retryable problem.
TransientErrorKind
} else if (ex.getErrorCode == 8176) {
// consistent read failure; rollback data not available
// Cause: Encountered data changed by an operation that does not generate rollback data
// Action: In read/write transactions, retry the intended operation.
TransientErrorKind
} else if (ex.getErrorCode == 8177) {
// failure to serialize transaction with serializable isolation level
TransientErrorKind
} else if (ex.getErrorCode == 17410) {
// No more data to read from socket, can be caused by network problems
SpuriousTransientErrorKind
} else if (code == 17002) {
// This has been observed as either IO Error: Connection reset by peer or IO Error: Broken pipe
// when straight-up killing an Oracle database server (`kill -9 <oracle-pid>`)
TransientErrorKind
} else if (code == 1088 || code == 1089 || code == 1090 || code == 1092) {
// Often observed for orderly Oracle shutdowns
// https://docs.oracle.com/en/database/oracle/oracle-database/19/errmg/ORA-00910.html#GUID-D9EBDFFA-88C6-4185-BD2C-E1B959A97274
TransientErrorKind
} else if (ex.getCause != null) {
logger.info("Unable to retry on exception, checking cause.")
retryOK(Failure(ex.getCause), logger)
} else {
FatalErrorKind
}
case _ => FatalErrorKind
case ff @ Failure(exception) =>
val errorKind = retryOKInternal(ff, logger)
// only log the full exception if the error kind changed such that we avoid spamming the logs
if (!lastErrorKind.contains(errorKind)) {
logThrowable(exception, logger)
} else {
logger.debug(
s"Retrying on same error kind ${errorKind} for ${exception.getClass.getSimpleName}/${exception.getMessage}"
)
}
errorKind
}
}
@tailrec private def retryOKInternal(
outcome: Failure[_],
logger: TracedLogger,
)(implicit
tc: TraceContext
): ErrorKind = {
outcome.exception match {
case exn: java.util.concurrent.RejectedExecutionException =>
// This occurs when slick's task queue is full
// Create a CantonError so that the error code gets logged.
DatabaseTaskRejected(exn.toString)(
ErrorLoggingContext.fromTracedLogger(logger)
).discard
TransientErrorKind
case exception: PSQLException =>
// Error codes documented here: https://www.postgresql.org/docs/9.6/errcodes-appendix.html
val error = exception.getSQLState
if (error.startsWith("08")) {
// Class 08 Connection Exception
TransientErrorKind
} else if (error == "40001") {
// Class 40 Transaction Rollback: 40001 serialization_failure
// Failure to serialize db accesses, happens due to contention
TransientErrorKind
} else if (error == "40P01") {
// Deadlock
// See DatabaseDeadlockTestPostgres
// This also covers deadlocks reported as BatchUpdateExceptions,
// because they refer to a PSQLException has cause.
TransientErrorKind
} else if (error == "25006") {
// Retry on read only transaction, which can occur on Azure
TransientErrorKind
} else if (error.startsWith("57P") && error != "57P014" && error != "57P04") {
// Retry on operator invention errors, otherwise Canton components crash in an uncontrolled manner when
// the exception bubbles up (don't retry on `query_canceled` and `database_dropped`)
TransientErrorKind
} else {
// Don't retry on other exceptions. These other exceptions should be those for which retrying typically won't
// help, for example a unique constraint violation.
FatalErrorKind
}
case _: SQLIntegrityConstraintViolationException =>
// Both H2 and Oracle may fail with spurious constraint violations, due to racy implementation of the MERGE statements.
// In H2, this may also occur because it does not properly implement the serializable isolation level.
// See UpsertTestOracle
// See https://github.com/h2database/h2database/issues/2167
SpuriousTransientErrorKind
case _: SQLRecoverableException | _: SQLTransientException |
_: SQLNonTransientConnectionException =>
TransientErrorKind
case ex: SQLException =>
val code = ex.getErrorCode
if (ex.getErrorCode == 1) {
// Retry on ORA-00001: unique constraint violated exception
SpuriousTransientErrorKind
} else if (ex.getMessage == "Connection is closed") {
// May fail with a "Connection is closed" message if the db has gone down
TransientErrorKind
} else if (ex.getErrorCode == 4021) {
// ORA timeout occurred while waiting to lock object
TransientErrorKind
} else if (ex.getErrorCode == 54) {
// ORA timeout occurred while waiting to lock object or because NOWAIT has been set
// e.g. as part of truncate table
TransientErrorKind
} else if (ex.getErrorCode == 60) {
// Deadlock
// See DatabaseDeadlockTestOracle
TransientErrorKind
} else if (
ex.getErrorCode == 604 &&
List("ORA-08176", "ORA-08177").exists(ex.getMessage.contains)
) {
// Oracle failure in a batch operation
// For Oracle, the `cause` is not always set properly for exceptions. This is a problem for batched queries.
// So, look through an exception's `message` to see if it contains a retryable problem.
TransientErrorKind
} else if (ex.getErrorCode == 8176) {
// consistent read failure; rollback data not available
// Cause: Encountered data changed by an operation that does not generate rollback data
// Action: In read/write transactions, retry the intended operation.
TransientErrorKind
} else if (ex.getErrorCode == 8177) {
// failure to serialize transaction with serializable isolation level
TransientErrorKind
} else if (ex.getErrorCode == 17410) {
// No more data to read from socket, can be caused by network problems
SpuriousTransientErrorKind
} else if (code == 17002) {
// This has been observed as either IO Error: Connection reset by peer or IO Error: Broken pipe
// when straight-up killing an Oracle database server (`kill -9 <oracle-pid>`)
TransientErrorKind
} else if (code == 1088 || code == 1089 || code == 1090 || code == 1092) {
// Often observed for orderly Oracle shutdowns
// https://docs.oracle.com/en/database/oracle/oracle-database/19/errmg/ORA-00910.html#GUID-D9EBDFFA-88C6-4185-BD2C-E1B959A97274
TransientErrorKind
} else if (ex.getCause != null) {
logger.info("Unable to retry on exception, checking cause.")
retryOKInternal(Failure(ex.getCause), logger)
} else {
FatalErrorKind
}
case _ => FatalErrorKind
}
}
}
@ -218,8 +238,8 @@ object RetryUtil {
*/
case object AllExnRetryable extends ExceptionRetryable {
override def retryOK(outcome: Try[_], logger: TracedLogger)(implicit
tc: TraceContext
override def retryOK(outcome: Try[_], logger: TracedLogger, lastErrorKind: Option[ErrorKind])(
implicit tc: TraceContext
): ErrorKind = {
outcome.forFailed(t => logThrowable(t, logger))
NoErrorKind
@ -231,8 +251,8 @@ object RetryUtil {
*/
case object NoExnRetryable extends ExceptionRetryable {
override def retryOK(outcome: Try[_], logger: TracedLogger)(implicit
tc: TraceContext
override def retryOK(outcome: Try[_], logger: TracedLogger, lastErrorKind: Option[ErrorKind])(
implicit tc: TraceContext
): ErrorKind = outcome match {
case Failure(ex) =>
logThrowable(ex, logger)

View File

@ -1,4 +1,4 @@
sdk-version: 2.8.0-snapshot.20231118.12382.0.v86cb8054
sdk-version: 2.8.0-snapshot.20231121.12391.0.v6eaf5081
build-options:
- --target=1.14
name: CantonExamples

View File

@ -60,8 +60,7 @@ trait DatabaseLimitNbParamTest
rawStorage
.update(query.asUpdate, "parameter limit query", maxRetries = 1)
.transformWith { outcome =>
val errorKind = DbExceptionRetryable.retryOK(outcome, logger)
val errorKind = DbExceptionRetryable.retryOK(outcome, logger, None)
errorKind match {
case FatalErrorKind =>
case _ => fail("Database error kind should be fatal")

View File

@ -1,4 +1,4 @@
sdk-version: 2.8.0-snapshot.20231118.12382.0.v86cb8054
sdk-version: 2.8.0-snapshot.20231121.12391.0.v6eaf5081
name: ai-analysis
source: AIAnalysis.daml
init-script: AIAnalysis:setup

View File

@ -1,4 +1,4 @@
sdk-version: 2.8.0-snapshot.20231118.12382.0.v86cb8054
sdk-version: 2.8.0-snapshot.20231121.12391.0.v6eaf5081
name: bank
source: Bank.daml
init-script: Bank:setup

View File

@ -1,4 +1,4 @@
sdk-version: 2.8.0-snapshot.20231118.12382.0.v86cb8054
sdk-version: 2.8.0-snapshot.20231121.12391.0.v6eaf5081
name: doctor
source: Doctor.daml
init-script: Doctor:setup

View File

@ -1,4 +1,4 @@
sdk-version: 2.8.0-snapshot.20231118.12382.0.v86cb8054
sdk-version: 2.8.0-snapshot.20231121.12391.0.v6eaf5081
name: health-insurance
source: HealthInsurance.daml
init-script: HealthInsurance:setup

View File

@ -1,4 +1,4 @@
sdk-version: 2.8.0-snapshot.20231118.12382.0.v86cb8054
sdk-version: 2.8.0-snapshot.20231121.12391.0.v6eaf5081
name: medical-records
source: MedicalRecord.daml
init-script: MedicalRecord:setup

View File

@ -1,4 +1,4 @@
sdk-version: 2.8.0-snapshot.20231118.12382.0.v86cb8054
sdk-version: 2.8.0-snapshot.20231121.12391.0.v6eaf5081
build-options:
- --target=1.14
name: JsonEncodingTest

View File

@ -1,4 +1,4 @@
sdk-version: 2.8.0-snapshot.20231118.12382.0.v86cb8054
sdk-version: 2.8.0-snapshot.20231121.12391.0.v6eaf5081
name: AdminWorkflows
source: AdminWorkflows.daml
version: 2.8.0

View File

@ -1,4 +1,4 @@
sdk-version: 2.8.0-snapshot.20231118.12382.0.v86cb8054
sdk-version: 2.8.0-snapshot.20231121.12391.0.v6eaf5081
build-options:
- --target=1.14
name: AdminWorkflowsWithVacuuming

View File

@ -23,9 +23,9 @@ if [ "{local}" = "true" ]; then
exit 0
fi
CANTON_ENTERPRISE_VERSION=2.8.0-snapshot.20231123.11580.0.v1dd021e2
CANTON_ENTERPRISE_SHA=c43c76cdbb0aca589f13af54ef676cffd4e68e1f4a506cf9a5822bb94d9f1987
CANTON_ENTERPRISE_URL=https://digitalasset.jfrog.io/artifactory/assembly/daml/canton-backup/2.8.0-snapshot.20231123.11580.0.v1dd021e2/c43c76cdbb0aca589f13af54ef676cffd4e68e1f4a506cf9a5822bb94d9f1987/canton-enterprise-2.8.0-snapshot.20231123.11580.0.v1dd021e2.tar.gz
CANTON_ENTERPRISE_VERSION=2.8.0-snapshot.20231124.11584.0.v33f28bad
CANTON_ENTERPRISE_SHA=543df892641b3c68b7f38d961c4404b5753b311cbf4cc3bf0b0d16f22d44a9a6
CANTON_ENTERPRISE_URL=https://digitalasset.jfrog.io/artifactory/assembly/daml/canton-backup/2.8.0-snapshot.20231124.11584.0.v33f28bad/543df892641b3c68b7f38d961c4404b5753b311cbf4cc3bf0b0d16f22d44a9a6/canton-enterprise-2.8.0-snapshot.20231124.11584.0.v33f28bad.tar.gz
url=$$CANTON_ENTERPRISE_URL