digital-asset/daml
1
1
Fork 0
mirror of https://github.com/digital-asset/daml.git synced 2024-09-20 09:17:43 +03:00
Code Issues Projects Releases Wiki Activity

Add string-interning data access [DPP-707] (#11484)

Browse Source 
* Add new table, and party_id to party_entries
* Add write side basics (DbDto, Schema)
* Add read side basic (StorageBackendStringInterning)
* Add StorageBackend unit test
* Support for all DB backends

changelog_begin
changelog_end
This commit is contained in:
Marton Nagy 2021-11-04 03:07:32 +01:00 committed by GitHub
parent 6c9416492d
commit 541d07c3a9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 174 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ledger/participant-integration-api/src/main/resources/db/migration/h2database-appendonly/V1__Append_only_schema.sha256
View File

@ -1 +1 @@
86ded6839752d76bb3e3e6cde7f4ec103dea6cee6382b6695de95e84dd74d2da
e28b88accecead16dcd67becabf7ad230654b65f93c277e785ba77203c3db8b5

5
ledger/participant-integration-api/src/main/resources/db/migration/h2database-appendonly/V1__Append_only_schema.sql
View File

@ -539,3 +539,8 @@ SELECT
exercise_result_compression
FROM participant_events_non_consuming_exercise
;
CREATE TABLE string_interning (
internal_id integer PRIMARY KEY NOT NULL,
external_string text
);

1
ledger/participant-integration-api/src/main/resources/db/migration/oracle-appendonly/V3__Add_string_interning.sha256 Normal file
View File

@ -0,0 +1 @@
f87eb3e6709109d9fa34a9757cf53e3f8b84584a079442eea6e8da4f4224dc2e

7
ledger/participant-integration-api/src/main/resources/db/migration/oracle-appendonly/V3__Add_string_interning.sql Normal file
View File

@ -0,0 +1,7 @@
-- Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
-- SPDX-License-Identifier: Apache-2.0
CREATE TABLE string_interning (
internal_id NUMBER PRIMARY KEY NOT NULL,
external_string VARCHAR2(4000)
);

1
ledger/participant-integration-api/src/main/resources/db/migration/postgres-appendonly/V112__add_string_interning.sha256 Normal file
View File

@ -0,0 +1 @@
d807d5d30cc3c3a55ad6a7319f21cc8a77f78c0a76fbdc94cd3c6a895e432fb7

7
ledger/participant-integration-api/src/main/resources/db/migration/postgres-appendonly/V112__add_string_interning.sql Normal file
View File

@ -0,0 +1,7 @@
-- Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
-- SPDX-License-Identifier: Apache-2.0
CREATE TABLE string_interning (
internal_id INTEGER PRIMARY KEY NOT NULL,
external_string TEXT
);

4
ledger/participant-integration-api/src/main/scala/platform/store/backend/DbDto.scala
View File

@ -135,4 +135,8 @@ object DbDto {
final case class CommandDeduplication(deduplication_key: String) extends DbDto
final case class StringInterningDto(
internalId: Int,
externalString: String,
) extends DbDto
}

7
ledger/participant-integration-api/src/main/scala/platform/store/backend/StorageBackend.scala
View File

@ -48,6 +48,7 @@ trait StorageBackend[DB_BATCH]
with DBLockStorageBackend
with IntegrityStorageBackend
with ResetStorageBackend
with StringInterningStorageBackend
trait ResetStorageBackend {
@ -352,6 +353,12 @@ trait IntegrityStorageBackend {
def verifyIntegrity()(connection: Connection): Unit
}
trait StringInterningStorageBackend {
def loadStringInterningEntries(fromIdExclusive: Int, untilIdInclusive: Int)(
connection: Connection
): Iterable[(Int, String)]
}
object StorageBackend {
case class RawContractState(
templateId: Option[String],

1
ledger/participant-integration-api/src/main/scala/platform/store/backend/StorageBackendFactory.scala
View File

@ -22,6 +22,7 @@ trait StorageBackendFactory {
def createDBLockStorageBackend: DBLockStorageBackend
def createIntegrityStorageBackend: IntegrityStorageBackend
def createResetStorageBackend: ResetStorageBackend
def createStringInterningStorageBackend: StringInterningStorageBackend
}
object StorageBackendFactory {

6
ledger/participant-integration-api/src/main/scala/platform/store/backend/common/Field.scala
View File

@ -65,9 +65,11 @@ private[backend] case class Bytea[FROM](extract: FROM => Array[Byte])
private[backend] case class ByteaOptional[FROM](extract: FROM => Option[Array[Byte]])
extends TrivialOptionalField[FROM, Array[Byte]]
private[backend] case class Integer[FROM](extract: FROM => Int) extends TrivialField[FROM, Int]
private[backend] case class IntOptional[FROM](extract: FROM => Option[Int])
extends Field[FROM, Option[Int], java.lang.Integer] {
override def convert: Option[Int] => Integer = _.map(Int.box).orNull
override def convert: Option[Int] => java.lang.Integer = _.map(Int.box).orNull
}
private[backend] case class Bigint[FROM](extract: FROM => Long) extends TrivialField[FROM, Long]
@ -79,7 +81,7 @@ private[backend] case class BigintOptional[FROM](extract: FROM => Option[Long])
private[backend] case class SmallintOptional[FROM](extract: FROM => Option[Int])
extends Field[FROM, Option[Int], java.lang.Integer] {
override def convert: Option[Int] => Integer = _.map(Int.box).orNull
override def convert: Option[Int] => java.lang.Integer = _.map(Int.box).orNull
}
private[backend] case class BooleanField[FROM](extract: FROM => Boolean)

11
ledger/participant-integration-api/src/main/scala/platform/store/backend/common/Schema.scala
View File

@ -52,6 +52,9 @@ private[backend] object AppendOnlySchema {
def smallintOptional[FROM, _](extractor: FROM => Option[Int]): Field[FROM, Option[Int], _] =
SmallintOptional(extractor)
def int[FROM, _](extractor: FROM => Int): Field[FROM, Int, _] =
Integer(extractor)
def intOptional[FROM, _](extractor: FROM => Option[Int]): Field[FROM, Option[Int], _] =
IntOptional(extractor)
@ -227,6 +230,12 @@ private[backend] object AppendOnlySchema {
"deduplication_key" -> fieldStrategy.string(_.deduplication_key)
)
val stringInterningTable: Table[DbDto.StringInterningDto] =
fieldStrategy.insert("string_interning")(
"internal_id" -> fieldStrategy.int(_.internalId),
"external_string" -> fieldStrategy.string(_.externalString),
)
val executes: Seq[Array[Array[_]] => Connection => Unit] = List(
eventsDivulgence.executeUpdate,
eventsCreate.executeUpdate,
@ -238,6 +247,7 @@ private[backend] object AppendOnlySchema {
partyEntries.executeUpdate,
commandCompletions.executeUpdate,
commandSubmissionDeletes.executeUpdate,
stringInterningTable.executeUpdate,
)
new Schema[DbDto] {
@ -257,6 +267,7 @@ private[backend] object AppendOnlySchema {
partyEntries.prepareData(collect[PartyEntry]),
commandCompletions.prepareData(collect[CommandCompletion]),
commandSubmissionDeletes.prepareData(collect[CommandDeduplication]),
stringInterningTable.prepareData(collect[StringInterningDto]),
)
}

30
ledger/participant-integration-api/src/main/scala/platform/store/backend/common/StringInterningStorageBackendTemplate.scala Normal file
View File

@ -0,0 +1,30 @@
// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package com.daml.platform.store.backend.common
import java.sql.Connection
import anorm.SqlParser.{int, str}
import anorm.{RowParser, SqlStringInterpolation, ~}
import com.daml.platform.store.backend.StringInterningStorageBackend
import com.daml.platform.store.SimpleSqlAsVectorOf.SimpleSqlAsVectorOf
object StringInterningStorageBackendTemplate extends StringInterningStorageBackend {
private val StringInterningEntriesParser: RowParser[(Int, String)] =
int("internal_id") ~ str("external_string") map { case internalId ~ externalString =>
(internalId, externalString)
}
override def loadStringInterningEntries(fromIdExclusive: Int, untilIdInclusive: Int)(
connection: Connection
): Iterable[(Int, String)] =
SQL"""
SELECT internal_id, external_string
FROM string_interning
WHERE
internal_id > $fromIdExclusive
AND internal_id <= $untilIdInclusive
""".asVectorOf(StringInterningEntriesParser)(connection)
}

2
ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2ResetStorageBackend.scala
View File

@ -22,6 +22,7 @@ object H2ResetStorageBackend extends ResetStorageBackend {
|truncate table participant_events_consuming_exercise;
|truncate table participant_events_non_consuming_exercise;
|truncate table party_entries;
|truncate table string_interning;
|set referential_integrity true;""".stripMargin)
.execute()(connection)
()
@ -40,6 +41,7 @@ object H2ResetStorageBackend extends ResetStorageBackend {
|truncate table participant_events_consuming_exercise;
|truncate table participant_events_non_consuming_exercise;
|truncate table party_entries;
|truncate table string_interning;
|set referential_integrity true;""".stripMargin)
.execute()(connection)
()

5
ledger/participant-integration-api/src/main/scala/platform/store/backend/h2/H2StorageBackendFactory.scala
View File

@ -11,6 +11,7 @@ import com.daml.platform.store.backend.common.{
PackageStorageBackendTemplate,
ParameterStorageBackendTemplate,
PartyStorageBackendTemplate,
StringInterningStorageBackendTemplate,
}
import com.daml.platform.store.backend.{
CompletionStorageBackend,
@ -27,6 +28,7 @@ import com.daml.platform.store.backend.{
PartyStorageBackend,
ResetStorageBackend,
StorageBackendFactory,
StringInterningStorageBackend,
}
object H2StorageBackendFactory extends StorageBackendFactory {
@ -68,4 +70,7 @@ object H2StorageBackendFactory extends StorageBackendFactory {
override val createResetStorageBackend: ResetStorageBackend =
H2ResetStorageBackend
override val createStringInterningStorageBackend: StringInterningStorageBackend =
StringInterningStorageBackendTemplate
}

2
ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleResetStorageBackend.scala
View File

@ -21,6 +21,7 @@ object OracleResetStorageBackend extends ResetStorageBackend {
"truncate table participant_events_consuming_exercise cascade",
"truncate table participant_events_non_consuming_exercise cascade",
"truncate table party_entries cascade",
"truncate table string_interning cascade",
).map(SQL(_)).foreach(_.execute()(connection))
override def resetAll(connection: Connection): Unit =
@ -36,5 +37,6 @@ object OracleResetStorageBackend extends ResetStorageBackend {
"truncate table participant_events_consuming_exercise cascade",
"truncate table participant_events_non_consuming_exercise cascade",
"truncate table party_entries cascade",
"truncate table string_interning cascade",
).map(SQL(_)).foreach(_.execute()(connection))
}

5
ledger/participant-integration-api/src/main/scala/platform/store/backend/oracle/OracleStorageBackendFactory.scala
View File

@ -12,6 +12,7 @@ import com.daml.platform.store.backend.common.{
PackageStorageBackendTemplate,
ParameterStorageBackendTemplate,
PartyStorageBackendTemplate,
StringInterningStorageBackendTemplate,
}
import com.daml.platform.store.backend.{
CompletionStorageBackend,
@ -28,6 +29,7 @@ import com.daml.platform.store.backend.{
PartyStorageBackend,
ResetStorageBackend,
StorageBackendFactory,
StringInterningStorageBackend,
}
object OracleStorageBackendFactory extends StorageBackendFactory {
@ -69,4 +71,7 @@ object OracleStorageBackendFactory extends StorageBackendFactory {
override val createResetStorageBackend: ResetStorageBackend =
OracleResetStorageBackend
override val createStringInterningStorageBackend: StringInterningStorageBackend =
StringInterningStorageBackendTemplate
}

2
ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PostgresResetStorageBackend.scala
View File

@ -20,6 +20,7 @@ object PostgresResetStorageBackend extends ResetStorageBackend {
|truncate table participant_events_consuming_exercise cascade;
|truncate table participant_events_non_consuming_exercise cascade;
|truncate table party_entries cascade;
|truncate table string_interning cascade;
|""".stripMargin)
.execute()(connection)
()
@ -37,6 +38,7 @@ object PostgresResetStorageBackend extends ResetStorageBackend {
|truncate table participant_events_consuming_exercise cascade;
|truncate table participant_events_non_consuming_exercise cascade;
|truncate table party_entries cascade;
|truncate table string_interning cascade;
|""".stripMargin)
.execute()(connection)
()

5
ledger/participant-integration-api/src/main/scala/platform/store/backend/postgresql/PostgresStorageBackendFactory.scala
View File

@ -12,6 +12,7 @@ import com.daml.platform.store.backend.common.{
PackageStorageBackendTemplate,
ParameterStorageBackendTemplate,
PartyStorageBackendTemplate,
StringInterningStorageBackendTemplate,
}
import com.daml.platform.store.backend.{
CompletionStorageBackend,
@ -28,6 +29,7 @@ import com.daml.platform.store.backend.{
PartyStorageBackend,
ResetStorageBackend,
StorageBackendFactory,
StringInterningStorageBackend,
}
object PostgresStorageBackendFactory extends StorageBackendFactory {
@ -69,4 +71,7 @@ object PostgresStorageBackendFactory extends StorageBackendFactory {
override val createResetStorageBackend: ResetStorageBackend =
PostgresResetStorageBackend
override val createStringInterningStorageBackend: StringInterningStorageBackend =
StringInterningStorageBackendTemplate
}

3
ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendSuite.scala
View File

@ -15,6 +15,7 @@ trait StorageBackendSuite
with StorageBackendTestsDBLockForSuite
with StorageBackendTestsIntegrity
with StorageBackendTestsDeduplication
with StorageBackendTestsTimestamps {
with StorageBackendTestsTimestamps
with StorageBackendTestsStringInterning {
this: AsyncFlatSpec =>
}

15
ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestsReset.scala
View File

@ -20,6 +20,8 @@ private[backend] trait StorageBackendTestsReset extends Matchers with StorageBac
backendFactory.createPackageStorageBackend
private val contractStorageBackend: ContractStorageBackend =
backendFactory.createContractStorageBackend
private val stringInterningStorageBackend: StringInterningStorageBackend =
backendFactory.createStringInterningStorageBackend
private val resetStorageBackend: ResetStorageBackend = backendFactory.createResetStorageBackend
behavior of "StorageBackend (reset)"
@ -34,6 +36,9 @@ private[backend] trait StorageBackendTestsReset extends Matchers with StorageBac
config <- executeSql(configurationStorageBackend.ledgerConfiguration)
packages <- executeSql(packageStorageBackend.lfPackages)
events <- executeSql(contractStorageBackend.contractStateEvents(0, Long.MaxValue))
stringInterningEntries <- executeSql(
stringInterningStorageBackend.loadStringInterningEntries(0, 1000)
)
} yield {
identity shouldBe None
end shouldBe None
@ -41,6 +46,7 @@ private[backend] trait StorageBackendTestsReset extends Matchers with StorageBac
packages shouldBe empty
events shouldBe empty
config shouldBe None
stringInterningEntries shouldBe empty
}
}
@ -73,6 +79,7 @@ private[backend] trait StorageBackendTestsReset extends Matchers with StorageBac
dtoExercise(offset(5), 2L, true, "#4"),
dtoDivulgence(Some(offset(5)), 3L, "#4"),
dtoCompletion(offset(5)),
DbDto.StringInterningDto(2, "2"),
)
for {
@ -94,6 +101,9 @@ private[backend] trait StorageBackendTestsReset extends Matchers with StorageBac
parties <- executeSql(partyStorageBackend.knownParties)
config <- executeSql(configurationStorageBackend.ledgerConfiguration)
packages <- executeSql(packageStorageBackend.lfPackages)
stringInterningEntries <- executeSql(
stringInterningStorageBackend.loadStringInterningEntries(0, 1000)
)
} yield {
identity shouldBe None
end shouldBe None
@ -101,6 +111,7 @@ private[backend] trait StorageBackendTestsReset extends Matchers with StorageBac
packages should not be empty // Note: reset() does not delete packages
events shouldBe empty
config shouldBe None
stringInterningEntries shouldBe empty
}
}
@ -141,6 +152,9 @@ private[backend] trait StorageBackendTestsReset extends Matchers with StorageBac
parties <- executeSql(partyStorageBackend.knownParties)
config <- executeSql(configurationStorageBackend.ledgerConfiguration)
packages <- executeSql(packageStorageBackend.lfPackages)
stringInterningEntries <- executeSql(
stringInterningStorageBackend.loadStringInterningEntries(0, 1000)
)
} yield {
identity shouldBe None
end shouldBe None
@ -148,6 +162,7 @@ private[backend] trait StorageBackendTestsReset extends Matchers with StorageBac
packages shouldBe empty // Note: resetAll() does delete packages
events shouldBe empty
config shouldBe None
stringInterningEntries shouldBe empty
}
}

57
ledger/participant-integration-api/src/test/lib/scala/platform/store/backend/StorageBackendTestsStringInterning.scala Normal file
View File

@ -0,0 +1,57 @@
// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
package com.daml.platform.store.backend
import org.scalatest.Inside
import org.scalatest.flatspec.AsyncFlatSpec
import org.scalatest.matchers.should.Matchers
private[backend] trait StorageBackendTestsStringInterning
extends Matchers
with Inside
with StorageBackendSpec {
this: AsyncFlatSpec =>
private val stringInterningStorageBackend = backendFactory.createStringInterningStorageBackend
behavior of "StorageBackend (StringInterning)"
it should "store and load string-interning entries" in {
val dtos = Vector(
DbDto.StringInterningDto(2, "a"),
DbDto.StringInterningDto(3, "b"),
DbDto.StringInterningDto(4, "c"),
DbDto.StringInterningDto(5, "d"),
)
for {
interningIdsBeforeBegin <- executeSql(
stringInterningStorageBackend.loadStringInterningEntries(0, 5)
)
_ <- executeSql(ingest(dtos, _))
interningIdsFull <- executeSql(stringInterningStorageBackend.loadStringInterningEntries(0, 5))
interningIdsOverFetch <- executeSql(
stringInterningStorageBackend.loadStringInterningEntries(0, 10)
)
interningIdsEmpty <- executeSql(
stringInterningStorageBackend.loadStringInterningEntries(5, 10)
)
interningIdsSubset <- executeSql(
stringInterningStorageBackend.loadStringInterningEntries(3, 10)
)
} yield {
val expectedFullList = List(
2 -> "a",
3 -> "b",
4 -> "c",
5 -> "d",
)
interningIdsBeforeBegin shouldBe Nil
interningIdsFull shouldBe expectedFullList
interningIdsOverFetch shouldBe expectedFullList
interningIdsEmpty shouldBe Nil
interningIdsSubset shouldBe expectedFullList.drop(2)
}
}
}