kvutils: Remove fingerprints from the pre-executing validator. [KVL-747] (#8296)

* kvutils: Remove fingerprints from the pre-executing validator. The submission validator doesn't care about fingerprints, it only needs to know about them in order to discard them from the state value. This introduces a new typeclass, `HasDamlStateValue`, which can be customized for various state value types to make submission validation generic. A little more code for now, but we're setting things up so it can be deleted later. CHANGELOG_BEGIN CHANGELOG_END * kvutils: Add more comments for `PreExecutingSubmissionValidator`.
2024-09-19 16:57:40 +03:00 · 2020-12-16 16:15:08 +01:00 · 2020-12-16 16:15:08 +01:00 · 021697fff7
commit 021697fff7
parent 2d511b449d
6 changed files with 128 additions and 97 deletions
--- a/ledger/ledger-on-memory/src/main/scala/com/daml/ledger/on/memory/InMemoryLedgerReaderWriter.scala
+++ b/ledger/ledger-on-memory/src/main/scala/com/daml/ledger/on/memory/InMemoryLedgerReaderWriter.scala
@ -17,7 +17,8 @@ import com.daml.ledger.participant.state.kvutils.{
  Bytes,
  Fingerprint,
  FingerprintPlaceholder,
-  KeyValueCommitting
+  KeyValueCommitting,
+  `DamlStateValue with Fingerprint has DamlStateValue`
 }
 import com.daml.ledger.participant.state.v1.{LedgerId, Offset, ParticipantId, SubmissionResult}
 import com.daml.ledger.resources.{Resource, ResourceContext, ResourceOwner}
@ -237,7 +238,7 @@ object InMemoryLedgerReaderWriter {
    val commitStrategy = new LogAppenderPreExecutingCommitStrategy(keySerializationStrategy)
    val valueToFingerprint: Option[Value] => Fingerprint =
      _.getOrElse(FingerprintPlaceholder)
-    val validator = new PreExecutingSubmissionValidator(keyValueCommitting, metrics, commitStrategy)
+    val validator = new PreExecutingSubmissionValidator(keyValueCommitting, commitStrategy, metrics)
    val committer = new PreExecutingValidatingCommitter(
      keySerializationStrategy,
      validator,
--- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/participant/state/kvutils/package.scala
+++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/participant/state/kvutils/package.scala
@ -4,6 +4,7 @@
 package com.daml.ledger.participant.state

 import com.daml.ledger.participant.state.kvutils.DamlKvutils.{DamlStateKey, DamlStateValue}
+import com.daml.ledger.validator.HasDamlStateValue
 import com.daml.metrics.MetricName
 import com.google.protobuf.ByteString

@ -42,4 +43,11 @@ package object kvutils {

  implicit val `Bytes Ordering`: Ordering[Bytes] = Ordering.by(_.asReadOnlyByteBuffer)

+  implicit object `DamlStateValue with Fingerprint has DamlStateValue`
+      extends HasDamlStateValue[(Option[DamlStateValue], Fingerprint)] {
+    override def damlStateValue(
+        value: (Option[DamlStateValue], Fingerprint)
+    ): Option[DamlStateValue] = value._1
+  }
+
 }
--- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/PreExecutingSubmissionValidator.scala
+++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/PreExecutingSubmissionValidator.scala
@ -3,24 +3,13 @@

 package com.daml.ledger.validator.preexecution

-import com.daml.ledger.participant.state.kvutils.DamlKvutils.{
-  DamlStateKey,
-  DamlStateValue,
-  DamlSubmission
-}
+import com.daml.ledger.participant.state.kvutils.DamlKvutils.{DamlStateKey, DamlSubmission}
 import com.daml.ledger.participant.state.kvutils.api.LedgerReader
-import com.daml.ledger.participant.state.kvutils.{
-  Bytes,
-  DamlStateMapWithFingerprints,
-  Envelope,
-  Fingerprint,
-  KeyValueCommitting
-}
+import com.daml.ledger.participant.state.kvutils.{Bytes, Envelope, KeyValueCommitting}
 import com.daml.ledger.participant.state.v1.ParticipantId
-import com.daml.ledger.validator.ValidationFailed
 import com.daml.ledger.validator.batch.BatchedSubmissionValidator
-import com.daml.ledger.validator.preexecution.PreExecutingSubmissionValidator._
 import com.daml.ledger.validator.reading.StateReader
+import com.daml.ledger.validator.{HasDamlStateValue, ValidationFailed}
 import com.daml.logging.{ContextualizedLogger, LoggingContext}
 import com.daml.metrics.{Metrics, Timed}

@ -29,23 +18,31 @@ import scala.concurrent.{ExecutionContext, Future}

 /**
  * Validator for pre-executing submissions.
+  *
+  * @param committer      Generates the pre-execution result from the submission.
+  * @param commitStrategy The strategy used to generate the data committed to the ledger.
+  * @param metrics        Records metrics.
+  * @tparam StateValue The type of the state persisted to the ledger. This must implement
+  *                    [[HasDamlStateValue]].
+  * @tparam ReadSet    The type of the read set generated by the `commitStrategy`.
+  * @tparam WriteSet   The type of the write set generated by the `commitStrategy`.
  */
-class PreExecutingSubmissionValidator[ReadSet, WriteSet](
+final class PreExecutingSubmissionValidator[StateValue, ReadSet, WriteSet](
    committer: KeyValueCommitting,
-    metrics: Metrics,
    commitStrategy: PreExecutingCommitStrategy[
      DamlStateKey,
-      (Option[DamlStateValue], Fingerprint),
+      StateValue,
      ReadSet,
      WriteSet,
    ],
-) {
+    metrics: Metrics,
+)(implicit hasDamlStateValue: HasDamlStateValue[StateValue]) {
  private val logger = ContextualizedLogger.get(getClass)

  def validate(
      submissionEnvelope: Bytes,
      submittingParticipantId: ParticipantId,
-      ledgerStateReader: DamlLedgerStateReaderWithFingerprints,
+      ledgerStateReader: StateReader[DamlStateKey, StateValue],
  )(
      implicit executionContext: ExecutionContext,
      loggingContext: LoggingContext,
@ -56,7 +53,7 @@ class PreExecutingSubmissionValidator[ReadSet, WriteSet](
      for {
        decodedSubmission <- decodeSubmission(submissionEnvelope)
        fetchedInputs <- fetchSubmissionInputs(decodedSubmission, ledgerStateReader)
-        inputState = fetchedInputs.mapValues(_._1)
+        inputState = fetchedInputs.mapValues(hasDamlStateValue.damlStateValue)
        preExecutionResult = committer.preExecuteSubmission(
          LedgerReader.DefaultConfiguration,
          decodedSubmission,
@ -116,8 +113,8 @@ class PreExecutingSubmissionValidator[ReadSet, WriteSet](

  private def fetchSubmissionInputs(
      submission: DamlSubmission,
-      ledgerStateReader: DamlLedgerStateReaderWithFingerprints,
-  )(implicit executionContext: ExecutionContext): Future[DamlStateMapWithFingerprints] = {
+      ledgerStateReader: StateReader[DamlStateKey, StateValue],
+  )(implicit executionContext: ExecutionContext): Future[Map[DamlStateKey, StateValue]] = {
    val inputKeys = submission.getInputDamlStateList.asScala
    Timed.timedAndTrackedFuture(
      metrics.daml.kvutils.submission.validator.fetchInputs,
@ -126,7 +123,7 @@ class PreExecutingSubmissionValidator[ReadSet, WriteSet](
        inputValues <- ledgerStateReader.read(inputKeys)

        nestedInputKeys = inputValues.collect {
-          case (Some(value), _) if value.hasContractKeyState =>
+          case HasDamlStateValue(value) if value.hasContractKeyState =>
            val contractId = value.getContractKeyState.getContractId
            DamlStateKey.newBuilder.setContractId(contractId).build
        }
@ -141,8 +138,3 @@ class PreExecutingSubmissionValidator[ReadSet, WriteSet](
    )
  }
 }
-
-object PreExecutingSubmissionValidator {
-  type DamlLedgerStateReaderWithFingerprints =
-    StateReader[DamlStateKey, (Option[DamlStateValue], Fingerprint)]
-}
--- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/PreExecutingValidatingCommitter.scala
+++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/preexecution/PreExecutingValidatingCommitter.scala
@ -40,7 +40,11 @@ import scala.util.{Failure, Success}
  */
 class PreExecutingValidatingCommitter[WriteSet](
    keySerializationStrategy: StateKeySerializationStrategy,
-    validator: PreExecutingSubmissionValidator[ReadSet, WriteSet],
+    validator: PreExecutingSubmissionValidator[
+      (Option[DamlStateValue], Fingerprint),
+      ReadSet,
+      WriteSet,
+    ],
    valueToFingerprint: Option[Value] => Fingerprint,
    postExecutionConflictDetector: PostExecutionConflictDetector[
      Key,
--- a/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/reading/HasDamlStateValue.scala
+++ b/ledger/participant-state/kvutils/src/main/scala/com/daml/ledger/validator/reading/HasDamlStateValue.scala
@ -0,0 +1,32 @@
+// Copyright (c) 2020 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package com.daml.ledger.validator
+
+import com.daml.ledger.participant.state.kvutils.DamlKvutils.DamlStateValue
+
+/**
+  * This typeclass signifies that implementor contains an optional DAML state value.
+  *
+  * Used by the [[com.daml.ledger.validator.preexecution.PreExecutingSubmissionValidator]].
+  */
+trait HasDamlStateValue[T] {
+  def damlStateValue(value: T): Option[DamlStateValue]
+}
+
+object HasDamlStateValue {
+  def unapply[T](
+      value: T
+  )(implicit hasDamlStateValue: HasDamlStateValue[T]): Option[DamlStateValue] =
+    hasDamlStateValue.damlStateValue(value)
+
+  implicit object `DamlStateValue has itself` extends HasDamlStateValue[DamlStateValue] {
+    override def damlStateValue(value: DamlStateValue): Option[DamlStateValue] =
+      Some(value)
+  }
+
+  implicit def `Option[T] has DamlStateValue if T has DamlStateValue`[T](
+      implicit hasDamlStateValue: HasDamlStateValue[T]
+  ): HasDamlStateValue[Option[T]] =
+    (value: Option[T]) => value.flatMap(hasDamlStateValue.damlStateValue)
+}
--- a/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/preexecution/PreExecutingSubmissionValidatorSpec.scala
+++ b/ledger/participant-state/kvutils/src/test/suite/scala/com/daml/ledger/validator/preexecution/PreExecutingSubmissionValidatorSpec.scala
@ -12,24 +12,20 @@ import com.daml.ledger.participant.state.kvutils.KeyValueCommitting.PreExecution
 import com.daml.ledger.participant.state.kvutils.{
  Bytes,
  DamlStateMap,
-  DamlStateMapWithFingerprints,
  Envelope,
-  Fingerprint,
-  FingerprintPlaceholder,
  KeyValueCommitting,
  TestHelpers
 }
 import com.daml.ledger.participant.state.v1.Configuration
+import com.daml.ledger.validator.HasDamlStateValue
 import com.daml.ledger.validator.TestHelper._
 import com.daml.ledger.validator.ValidationFailed.ValidationError
-import com.daml.ledger.validator.preexecution.PreExecutingSubmissionValidator.DamlLedgerStateReaderWithFingerprints
 import com.daml.ledger.validator.preexecution.PreExecutingSubmissionValidatorSpec._
-import com.daml.ledger.validator.preexecution.PreExecutionCommitResult.ReadSet
+import com.daml.ledger.validator.reading.StateReader
 import com.daml.lf.data.Ref.ParticipantId
 import com.daml.lf.data.Time.Timestamp
 import com.daml.logging.LoggingContext
 import com.daml.metrics.Metrics
-import com.google.protobuf.ByteString
 import org.mockito.{ArgumentMatchersSugar, MockitoSugar}
 import org.scalatest.Assertion
 import org.scalatest.matchers.should.Matchers
@ -43,17 +39,14 @@ class PreExecutingSubmissionValidatorSpec extends AsyncWordSpec with Matchers wi

  "validate" should {
    "generate correct output in case of success" in {
-      val expectedReadSet = Map(
-        allDamlStateKeyTypes.head -> FingerprintPlaceholder
-      )
+      val expectedReadSet = Set(allDamlStateKeyTypes.head)
      val actualInputState = Map(
-        allDamlStateKeyTypes.head ->
-          (Some(DamlStateValue.getDefaultInstance) -> FingerprintPlaceholder)
+        allDamlStateKeyTypes.head -> Some(DamlStateValue.getDefaultInstance)
      )
      val expectedMinRecordTime = Some(recordTime.toInstant.minusSeconds(123))
      val expectedMaxRecordTime = Some(recordTime.toInstant)
-      val expectedSuccessWriteSet = ByteString.copyFromUtf8("success")
-      val expectedOutOfTimeBoundsWriteSet = ByteString.copyFromUtf8("failure")
+      val expectedSuccessWriteSet = TestWriteSet("success")
+      val expectedOutOfTimeBoundsWriteSet = TestWriteSet("failure")
      val expectedInvolvedParticipants = Set(TestHelpers.mkParticipantId(0))
      val instance = createInstance(
        expectedReadSet = expectedReadSet,
@ -66,56 +59,47 @@ class PreExecutingSubmissionValidatorSpec extends AsyncWordSpec with Matchers wi
      val ledgerStateReader = createLedgerStateReader(actualInputState)

      instance
-        .validate(anEnvelope(expectedReadSet.keySet), aParticipantId, ledgerStateReader)
+        .validate(anEnvelope(expectedReadSet), aParticipantId, ledgerStateReader)
        .map { actual =>
          actual.minRecordTime shouldBe expectedMinRecordTime
          actual.maxRecordTime shouldBe expectedMaxRecordTime
          actual.successWriteSet shouldBe expectedSuccessWriteSet
          actual.outOfTimeBoundsWriteSet shouldBe expectedOutOfTimeBoundsWriteSet
-          actual.readSet should have size expectedReadSet.size.toLong
+          actual.readSet shouldBe TestReadSet(expectedReadSet)
          actual.involvedParticipants shouldBe expectedInvolvedParticipants
        }
    }

    "return a sorted read set with correct fingerprints" in {
-      val expectedReadSet =
-        allDamlStateKeyTypes.map(key => key -> key.toByteString).toMap
+      val expectedReadSet = allDamlStateKeyTypes.toSet
      val actualInputState =
-        allDamlStateKeyTypes
-          .map(key => key -> ((Some(DamlStateValue.getDefaultInstance), key.toByteString)))
-          .toMap
+        allDamlStateKeyTypes.map(key => key -> Some(DamlStateValue.getDefaultInstance)).toMap
      val instance = createInstance(expectedReadSet = expectedReadSet)
      val ledgerStateReader = createLedgerStateReader(actualInputState)

      instance
-        .validate(anEnvelope(expectedReadSet.keySet), aParticipantId, ledgerStateReader)
+        .validate(anEnvelope(expectedReadSet), aParticipantId, ledgerStateReader)
        .map(verifyReadSet(_, expectedReadSet))
    }

    "return a read set when the contract keys are inconsistent" in {
      val contractKeyStateKey = makeContractKeyStateKey("id")
-      val contractKeyFingerprint = fingerprint("contract key")

      // At the time of pre-execution, the key points to contract A.
      val contractIdAStateKey = makeContractIdStateKey("contract ID A")
      val contractIdAStateValue = makeContractIdStateValue()
-      val contractIdAFingerprint = fingerprint("contract ID A")

      // However, at the time of validation, it points to contract B.
      val contractKeyBStateValue = makeContractKeyStateValue("contract ID B")
      val contractIdBStateKey = makeContractIdStateKey("contract ID B")
      val contractIdBStateValue = makeContractIdStateValue()
-      val contractIdBFingerprint = fingerprint("contract ID B")

      val preExecutedInputKeys = Set(contractKeyStateKey, contractIdAStateKey)
-      val expectedReadSet = Map(
-        contractKeyStateKey -> contractKeyFingerprint,
-        contractIdBStateKey -> contractIdBFingerprint,
-      )
+      val expectedReadSet = Set(contractKeyStateKey, contractIdBStateKey)
      val actualInputState = Map(
-        contractKeyStateKey -> ((Some(contractKeyBStateValue), contractKeyFingerprint)),
-        contractIdAStateKey -> ((Some(contractIdAStateValue), contractIdAFingerprint)),
-        contractIdBStateKey -> ((Some(contractIdBStateValue), contractIdBFingerprint)),
+        contractKeyStateKey -> Some(contractKeyBStateValue),
+        contractIdAStateKey -> Some(contractIdAStateValue),
+        contractIdBStateKey -> Some(contractIdBStateValue),
      )
      val instance = createInstance(expectedReadSet = expectedReadSet)
      val ledgerStateReader = createLedgerStateReader(actualInputState)
@ -138,7 +122,8 @@ class PreExecutingSubmissionValidatorSpec extends AsyncWordSpec with Matchers wi
        .validate(
          Envelope.enclose(aBatchedSubmission),
          aParticipantId,
-          mock[DamlLedgerStateReaderWithFingerprints])
+          mock[StateReader[DamlStateKey, TestValue]],
+        )
        .failed
        .map {
          case ValidationError(actualReason) =>
@ -150,7 +135,7 @@ class PreExecutingSubmissionValidatorSpec extends AsyncWordSpec with Matchers wi
      val instance = createInstance()

      instance
-        .validate(anInvalidEnvelope, aParticipantId, mock[DamlLedgerStateReaderWithFingerprints])
+        .validate(anInvalidEnvelope, aParticipantId, mock[StateReader[DamlStateKey, TestValue]])
        .failed
        .map {
          case ValidationError(actualReason) =>
@ -166,7 +151,8 @@ class PreExecutingSubmissionValidatorSpec extends AsyncWordSpec with Matchers wi
        .validate(
          anEnvelopedDamlLogEntry,
          aParticipantId,
-          mock[DamlLedgerStateReaderWithFingerprints])
+          mock[StateReader[DamlStateKey, TestValue]],
+        )
        .failed
        .map {
          case ValidationError(actualReason) =>
@ -185,6 +171,18 @@ object PreExecutingSubmissionValidatorSpec {

  private val metrics = new Metrics(new MetricRegistry)

+  private final case class TestValue(value: Option[DamlStateValue])
+
+  private object TestValue {
+    implicit object `TestValue has DamlStateValue` extends HasDamlStateValue[TestValue] {
+      override def damlStateValue(value: TestValue): Option[DamlStateValue] = value.value
+    }
+  }
+
+  private final case class TestReadSet(keys: Set[DamlStateKey])
+
+  private final case class TestWriteSet(value: String)
+
  private def anEnvelope(expectedReadSet: Set[DamlStateKey] = Set.empty): Bytes = {
    val submission = DamlSubmission
      .newBuilder()
@ -195,16 +193,16 @@ object PreExecutingSubmissionValidatorSpec {
  }

  private def createInstance(
-      expectedReadSet: Map[DamlStateKey, Fingerprint] = Map.empty,
+      expectedReadSet: Set[DamlStateKey] = Set.empty,
      expectedMinRecordTime: Option[Instant] = None,
      expectedMaxRecordTime: Option[Instant] = None,
-      expectedSuccessWriteSet: Bytes = ByteString.EMPTY,
-      expectedOutOfTimeBoundsWriteSet: Bytes = ByteString.EMPTY,
+      expectedSuccessWriteSet: TestWriteSet = TestWriteSet(""),
+      expectedOutOfTimeBoundsWriteSet: TestWriteSet = TestWriteSet(""),
      expectedInvolvedParticipants: Set[ParticipantId] = Set.empty,
-  ): PreExecutingSubmissionValidator[ReadSet, Bytes] = {
+  ): PreExecutingSubmissionValidator[TestValue, TestReadSet, TestWriteSet] = {
    val mockCommitter = mock[KeyValueCommitting]
    val result = PreExecutionResult(
-      expectedReadSet.keySet,
+      expectedReadSet,
      aLogEntry,
      Map.empty,
      aLogEntry,
@ -222,54 +220,50 @@ object PreExecutingSubmissionValidatorSpec {

    val mockCommitStrategy = mock[PreExecutingCommitStrategy[
      DamlStateKey,
-      (Option[DamlStateValue], Fingerprint),
-      ReadSet,
-      Bytes,
+      TestValue,
+      TestReadSet,
+      TestWriteSet,
    ]]
    when(
-      mockCommitStrategy.generateReadSet(any[DamlStateMapWithFingerprints], any[Set[DamlStateKey]]))
-      .thenAnswer[DamlStateMapWithFingerprints, Set[DamlStateKey]] {
-        (fetchedInputs, accessedKeys) =>
-          accessedKeys.map { key =>
-            val (_, fingerprint) = fetchedInputs(key)
-            key.toByteString -> fingerprint
-          }.toSeq
+      mockCommitStrategy.generateReadSet(any[Map[DamlStateKey, TestValue]], any[Set[DamlStateKey]]))
+      .thenAnswer[Map[DamlStateKey, TestValue], Set[DamlStateKey]] { (_, accessedKeys) =>
+        TestReadSet(accessedKeys)
      }
    when(
      mockCommitStrategy.generateWriteSets(
        eqTo(aParticipantId),
        any[DamlLogEntryId],
-        any[Map[DamlStateKey, (Option[DamlStateValue], Fingerprint)]],
+        any[Map[DamlStateKey, TestValue]],
        same(result),
      )(any[ExecutionContext]))
      .thenReturn(
        Future.successful(
-          PreExecutionCommitResult[Bytes](
+          PreExecutionCommitResult(
            expectedSuccessWriteSet,
            expectedOutOfTimeBoundsWriteSet,
-            expectedInvolvedParticipants)))
+            expectedInvolvedParticipants,
+          )))

-    new PreExecutingSubmissionValidator[ReadSet, Bytes](mockCommitter, metrics, mockCommitStrategy)
+    new PreExecutingSubmissionValidator(mockCommitter, mockCommitStrategy, metrics)
  }

  private def createLedgerStateReader(
-      inputState: Map[DamlStateKey, (Option[DamlStateValue], Fingerprint)]
-  ): DamlLedgerStateReaderWithFingerprints =
-    new DamlLedgerStateReaderWithFingerprints {
-      override def read(keys: Seq[DamlStateKey])(implicit executionContext: ExecutionContext)
-        : Future[Seq[(Option[DamlStateValue], Fingerprint)]] =
-        Future.successful(keys.map(inputState))
+      inputState: Map[DamlStateKey, Option[DamlStateValue]]
+  ): StateReader[DamlStateKey, TestValue] = {
+    val wrappedInputState = inputState.mapValues(TestValue(_))
+    new StateReader[DamlStateKey, TestValue] {
+      override def read(
+          keys: Seq[DamlStateKey]
+      )(implicit executionContext: ExecutionContext): Future[Seq[TestValue]] =
+        Future.successful(keys.map(wrappedInputState))
    }
+  }

  private def verifyReadSet(
-      output: PreExecutionOutput[ReadSet, Bytes],
-      expectedReadSet: Map[DamlStateKey, Fingerprint],
+      output: PreExecutionOutput[TestReadSet, Any],
+      expectedReadSet: Set[DamlStateKey],
  ): Assertion = {
    import org.scalatest.matchers.should.Matchers._
-    val actualReadSet = output.readSet.map {
-      case (key, value) =>
-        DamlStateKey.parseFrom(key) -> value
-    }.toMap
-    actualReadSet should be(expectedReadSet)
+    output.readSet.keys should be(expectedReadSet)
  }
 }