daml/ledger/ledger-on-sql/BUILD.bazel

216 lines
7.0 KiB
Python
Raw Normal View History

# Copyright (c) 2020 The DAML Authors. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
load(
"//bazel_tools:scala.bzl",
"da_scala_binary",
"da_scala_library",
"da_scala_test_suite",
)
load("//ledger/ledger-api-test-tool:conformance.bzl", "conformance_test")
supported_databases = [
{
ledger-on-sql: Get H2 working in the conformance tests. (#4155) * kvutils: Make logback.xml a base file, rather than the only option. * kvutils/app: Simplify logback.base.xml. Mostly by getting rid of unnecessary appenders. * ledger-on-sql: Add trace logging for all database work. * Upgrade H2; there's a few useful bug fixes. * kvutils/app: Let the user override the server JDBC URL. * kvutils/app: Provide a way to specify the ledger ID. * ledger-on-sql: If there are missing entries in the log, fail on read. This can happen right now because we insert in parallel. Next step: stop doing that. * ledger-on-sql: Stop writing in parallel; it causes race conditions. This unfortunately means we also stop _reading_ in parallel, which is less fun. * Revert "ledger-on-sql: Use a sequential log entry ID." This reverts commit c58265bf43b96d16bfa62ed30a795f365b1e83f2. * ledger-on-sql: Create tables with columns that are not nullable. * ledger-on-sql: The H2 conformance tests now work, just slowly. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Run tests against H2 and SQLite on memory and disk. * ledger-on-sql: Allow H2 to read from the log in parallel with writes. * ledger-on-sql: Use UUIDs for entry IDs, not 8 random bytes. * ledger-on-sql: Make sure to log the correlation ID. * ledger-on-sql: Do less while holding a database connection. * ledger-on-sql: Log the connection. * ledger-on-sql: We don't need to tell H2 not to drop the connection. The connection pool takes care of that. * ledger-on-sql: Disable H2 conformance tests on CI; they're too slow. * ledger-on-sql: Rename `loggingContext` to `logCtx`. * ledger-on-sql: Don't abuse log contexts; put the data in the message. * ledger-on-sql: Make the connection log line easier to read. And pull out the logger, even if it's only used once. * ledger-on-sql: Run the unit tests against (H2, SQLite) * (memory, file).
2020-01-22 18:54:51 +03:00
"name": "h2-memory",
"runtime_deps": [
"@maven//:com_h2database_h2",
],
"conformance_test_tags": [
"manual",
],
"conformance_test_server_args": [
ledger-on-sql: Get H2 working in the conformance tests. (#4155) * kvutils: Make logback.xml a base file, rather than the only option. * kvutils/app: Simplify logback.base.xml. Mostly by getting rid of unnecessary appenders. * ledger-on-sql: Add trace logging for all database work. * Upgrade H2; there's a few useful bug fixes. * kvutils/app: Let the user override the server JDBC URL. * kvutils/app: Provide a way to specify the ledger ID. * ledger-on-sql: If there are missing entries in the log, fail on read. This can happen right now because we insert in parallel. Next step: stop doing that. * ledger-on-sql: Stop writing in parallel; it causes race conditions. This unfortunately means we also stop _reading_ in parallel, which is less fun. * Revert "ledger-on-sql: Use a sequential log entry ID." This reverts commit c58265bf43b96d16bfa62ed30a795f365b1e83f2. * ledger-on-sql: Create tables with columns that are not nullable. * ledger-on-sql: The H2 conformance tests now work, just slowly. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Run tests against H2 and SQLite on memory and disk. * ledger-on-sql: Allow H2 to read from the log in parallel with writes. * ledger-on-sql: Use UUIDs for entry IDs, not 8 random bytes. * ledger-on-sql: Make sure to log the correlation ID. * ledger-on-sql: Do less while holding a database connection. * ledger-on-sql: Log the connection. * ledger-on-sql: We don't need to tell H2 not to drop the connection. The connection pool takes care of that. * ledger-on-sql: Disable H2 conformance tests on CI; they're too slow. * ledger-on-sql: Rename `loggingContext` to `logCtx`. * ledger-on-sql: Don't abuse log contexts; put the data in the message. * ledger-on-sql: Make the connection log line easier to read. And pull out the logger, even if it's only used once. * ledger-on-sql: Run the unit tests against (H2, SQLite) * (memory, file).
2020-01-22 18:54:51 +03:00
"--jdbc-url=jdbc:h2:mem:daml-on-sql-conformance-test",
],
"conformance_test_tool_args": [
"--concurrent-test-runs=2",
"--timeout-scale-factor=4",
],
},
{
ledger-on-sql: Get H2 working in the conformance tests. (#4155) * kvutils: Make logback.xml a base file, rather than the only option. * kvutils/app: Simplify logback.base.xml. Mostly by getting rid of unnecessary appenders. * ledger-on-sql: Add trace logging for all database work. * Upgrade H2; there's a few useful bug fixes. * kvutils/app: Let the user override the server JDBC URL. * kvutils/app: Provide a way to specify the ledger ID. * ledger-on-sql: If there are missing entries in the log, fail on read. This can happen right now because we insert in parallel. Next step: stop doing that. * ledger-on-sql: Stop writing in parallel; it causes race conditions. This unfortunately means we also stop _reading_ in parallel, which is less fun. * Revert "ledger-on-sql: Use a sequential log entry ID." This reverts commit c58265bf43b96d16bfa62ed30a795f365b1e83f2. * ledger-on-sql: Create tables with columns that are not nullable. * ledger-on-sql: The H2 conformance tests now work, just slowly. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Run tests against H2 and SQLite on memory and disk. * ledger-on-sql: Allow H2 to read from the log in parallel with writes. * ledger-on-sql: Use UUIDs for entry IDs, not 8 random bytes. * ledger-on-sql: Make sure to log the correlation ID. * ledger-on-sql: Do less while holding a database connection. * ledger-on-sql: Log the connection. * ledger-on-sql: We don't need to tell H2 not to drop the connection. The connection pool takes care of that. * ledger-on-sql: Disable H2 conformance tests on CI; they're too slow. * ledger-on-sql: Rename `loggingContext` to `logCtx`. * ledger-on-sql: Don't abuse log contexts; put the data in the message. * ledger-on-sql: Make the connection log line easier to read. And pull out the logger, even if it's only used once. * ledger-on-sql: Run the unit tests against (H2, SQLite) * (memory, file).
2020-01-22 18:54:51 +03:00
"name": "h2-file",
"runtime_deps": [
"@maven//:com_h2database_h2",
],
ledger-on-sql: Get H2 working in the conformance tests. (#4155) * kvutils: Make logback.xml a base file, rather than the only option. * kvutils/app: Simplify logback.base.xml. Mostly by getting rid of unnecessary appenders. * ledger-on-sql: Add trace logging for all database work. * Upgrade H2; there's a few useful bug fixes. * kvutils/app: Let the user override the server JDBC URL. * kvutils/app: Provide a way to specify the ledger ID. * ledger-on-sql: If there are missing entries in the log, fail on read. This can happen right now because we insert in parallel. Next step: stop doing that. * ledger-on-sql: Stop writing in parallel; it causes race conditions. This unfortunately means we also stop _reading_ in parallel, which is less fun. * Revert "ledger-on-sql: Use a sequential log entry ID." This reverts commit c58265bf43b96d16bfa62ed30a795f365b1e83f2. * ledger-on-sql: Create tables with columns that are not nullable. * ledger-on-sql: The H2 conformance tests now work, just slowly. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Run tests against H2 and SQLite on memory and disk. * ledger-on-sql: Allow H2 to read from the log in parallel with writes. * ledger-on-sql: Use UUIDs for entry IDs, not 8 random bytes. * ledger-on-sql: Make sure to log the correlation ID. * ledger-on-sql: Do less while holding a database connection. * ledger-on-sql: Log the connection. * ledger-on-sql: We don't need to tell H2 not to drop the connection. The connection pool takes care of that. * ledger-on-sql: Disable H2 conformance tests on CI; they're too slow. * ledger-on-sql: Rename `loggingContext` to `logCtx`. * ledger-on-sql: Don't abuse log contexts; put the data in the message. * ledger-on-sql: Make the connection log line easier to read. And pull out the logger, even if it's only used once. * ledger-on-sql: Run the unit tests against (H2, SQLite) * (memory, file).
2020-01-22 18:54:51 +03:00
"conformance_test_tags": [
"manual",
],
"conformance_test_server_main": "com.daml.ledger.on.sql.MainWithEphemeralDirectory",
"conformance_test_server_args": [
"--jdbc-url=jdbc:h2:%DIR/test",
],
"conformance_test_tool_args": [
"--concurrent-test-runs=2",
"--timeout-scale-factor=4",
],
},
{
"name": "postgresql",
ledger-on-sql: Get H2 working in the conformance tests. (#4155) * kvutils: Make logback.xml a base file, rather than the only option. * kvutils/app: Simplify logback.base.xml. Mostly by getting rid of unnecessary appenders. * ledger-on-sql: Add trace logging for all database work. * Upgrade H2; there's a few useful bug fixes. * kvutils/app: Let the user override the server JDBC URL. * kvutils/app: Provide a way to specify the ledger ID. * ledger-on-sql: If there are missing entries in the log, fail on read. This can happen right now because we insert in parallel. Next step: stop doing that. * ledger-on-sql: Stop writing in parallel; it causes race conditions. This unfortunately means we also stop _reading_ in parallel, which is less fun. * Revert "ledger-on-sql: Use a sequential log entry ID." This reverts commit c58265bf43b96d16bfa62ed30a795f365b1e83f2. * ledger-on-sql: Create tables with columns that are not nullable. * ledger-on-sql: The H2 conformance tests now work, just slowly. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Run tests against H2 and SQLite on memory and disk. * ledger-on-sql: Allow H2 to read from the log in parallel with writes. * ledger-on-sql: Use UUIDs for entry IDs, not 8 random bytes. * ledger-on-sql: Make sure to log the correlation ID. * ledger-on-sql: Do less while holding a database connection. * ledger-on-sql: Log the connection. * ledger-on-sql: We don't need to tell H2 not to drop the connection. The connection pool takes care of that. * ledger-on-sql: Disable H2 conformance tests on CI; they're too slow. * ledger-on-sql: Rename `loggingContext` to `logCtx`. * ledger-on-sql: Don't abuse log contexts; put the data in the message. * ledger-on-sql: Make the connection log line easier to read. And pull out the logger, even if it's only used once. * ledger-on-sql: Run the unit tests against (H2, SQLite) * (memory, file).
2020-01-22 18:54:51 +03:00
"runtime_deps": [
"@maven//:org_postgresql_postgresql",
ledger-on-sql: Get H2 working in the conformance tests. (#4155) * kvutils: Make logback.xml a base file, rather than the only option. * kvutils/app: Simplify logback.base.xml. Mostly by getting rid of unnecessary appenders. * ledger-on-sql: Add trace logging for all database work. * Upgrade H2; there's a few useful bug fixes. * kvutils/app: Let the user override the server JDBC URL. * kvutils/app: Provide a way to specify the ledger ID. * ledger-on-sql: If there are missing entries in the log, fail on read. This can happen right now because we insert in parallel. Next step: stop doing that. * ledger-on-sql: Stop writing in parallel; it causes race conditions. This unfortunately means we also stop _reading_ in parallel, which is less fun. * Revert "ledger-on-sql: Use a sequential log entry ID." This reverts commit c58265bf43b96d16bfa62ed30a795f365b1e83f2. * ledger-on-sql: Create tables with columns that are not nullable. * ledger-on-sql: The H2 conformance tests now work, just slowly. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Run tests against H2 and SQLite on memory and disk. * ledger-on-sql: Allow H2 to read from the log in parallel with writes. * ledger-on-sql: Use UUIDs for entry IDs, not 8 random bytes. * ledger-on-sql: Make sure to log the correlation ID. * ledger-on-sql: Do less while holding a database connection. * ledger-on-sql: Log the connection. * ledger-on-sql: We don't need to tell H2 not to drop the connection. The connection pool takes care of that. * ledger-on-sql: Disable H2 conformance tests on CI; they're too slow. * ledger-on-sql: Rename `loggingContext` to `logCtx`. * ledger-on-sql: Don't abuse log contexts; put the data in the message. * ledger-on-sql: Make the connection log line easier to read. And pull out the logger, even if it's only used once. * ledger-on-sql: Run the unit tests against (H2, SQLite) * (memory, file).
2020-01-22 18:54:51 +03:00
],
"conformance_test_server_main": "com.daml.ledger.on.sql.MainWithEphemeralPostgresql",
ledger-on-sql: Get H2 working in the conformance tests. (#4155) * kvutils: Make logback.xml a base file, rather than the only option. * kvutils/app: Simplify logback.base.xml. Mostly by getting rid of unnecessary appenders. * ledger-on-sql: Add trace logging for all database work. * Upgrade H2; there's a few useful bug fixes. * kvutils/app: Let the user override the server JDBC URL. * kvutils/app: Provide a way to specify the ledger ID. * ledger-on-sql: If there are missing entries in the log, fail on read. This can happen right now because we insert in parallel. Next step: stop doing that. * ledger-on-sql: Stop writing in parallel; it causes race conditions. This unfortunately means we also stop _reading_ in parallel, which is less fun. * Revert "ledger-on-sql: Use a sequential log entry ID." This reverts commit c58265bf43b96d16bfa62ed30a795f365b1e83f2. * ledger-on-sql: Create tables with columns that are not nullable. * ledger-on-sql: The H2 conformance tests now work, just slowly. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Run tests against H2 and SQLite on memory and disk. * ledger-on-sql: Allow H2 to read from the log in parallel with writes. * ledger-on-sql: Use UUIDs for entry IDs, not 8 random bytes. * ledger-on-sql: Make sure to log the correlation ID. * ledger-on-sql: Do less while holding a database connection. * ledger-on-sql: Log the connection. * ledger-on-sql: We don't need to tell H2 not to drop the connection. The connection pool takes care of that. * ledger-on-sql: Disable H2 conformance tests on CI; they're too slow. * ledger-on-sql: Rename `loggingContext` to `logCtx`. * ledger-on-sql: Don't abuse log contexts; put the data in the message. * ledger-on-sql: Make the connection log line easier to read. And pull out the logger, even if it's only used once. * ledger-on-sql: Run the unit tests against (H2, SQLite) * (memory, file).
2020-01-22 18:54:51 +03:00
},
{
"name": "sqlite-memory",
"runtime_deps": [
"@maven//:org_xerial_sqlite_jdbc",
],
"conformance_test_server_args": [
"--jdbc-url=jdbc:sqlite:file:daml-on-sql-conformance-test?mode=memory&cache=shared",
],
ledger-on-sql: Get H2 working in the conformance tests. (#4155) * kvutils: Make logback.xml a base file, rather than the only option. * kvutils/app: Simplify logback.base.xml. Mostly by getting rid of unnecessary appenders. * ledger-on-sql: Add trace logging for all database work. * Upgrade H2; there's a few useful bug fixes. * kvutils/app: Let the user override the server JDBC URL. * kvutils/app: Provide a way to specify the ledger ID. * ledger-on-sql: If there are missing entries in the log, fail on read. This can happen right now because we insert in parallel. Next step: stop doing that. * ledger-on-sql: Stop writing in parallel; it causes race conditions. This unfortunately means we also stop _reading_ in parallel, which is less fun. * Revert "ledger-on-sql: Use a sequential log entry ID." This reverts commit c58265bf43b96d16bfa62ed30a795f365b1e83f2. * ledger-on-sql: Create tables with columns that are not nullable. * ledger-on-sql: The H2 conformance tests now work, just slowly. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Run tests against H2 and SQLite on memory and disk. * ledger-on-sql: Allow H2 to read from the log in parallel with writes. * ledger-on-sql: Use UUIDs for entry IDs, not 8 random bytes. * ledger-on-sql: Make sure to log the correlation ID. * ledger-on-sql: Do less while holding a database connection. * ledger-on-sql: Log the connection. * ledger-on-sql: We don't need to tell H2 not to drop the connection. The connection pool takes care of that. * ledger-on-sql: Disable H2 conformance tests on CI; they're too slow. * ledger-on-sql: Rename `loggingContext` to `logCtx`. * ledger-on-sql: Don't abuse log contexts; put the data in the message. * ledger-on-sql: Make the connection log line easier to read. And pull out the logger, even if it's only used once. * ledger-on-sql: Run the unit tests against (H2, SQLite) * (memory, file).
2020-01-22 18:54:51 +03:00
},
{
"name": "sqlite-file",
"runtime_deps": [
"@maven//:org_xerial_sqlite_jdbc",
],
ledger-on-sql: Get H2 working in the conformance tests. (#4155) * kvutils: Make logback.xml a base file, rather than the only option. * kvutils/app: Simplify logback.base.xml. Mostly by getting rid of unnecessary appenders. * ledger-on-sql: Add trace logging for all database work. * Upgrade H2; there's a few useful bug fixes. * kvutils/app: Let the user override the server JDBC URL. * kvutils/app: Provide a way to specify the ledger ID. * ledger-on-sql: If there are missing entries in the log, fail on read. This can happen right now because we insert in parallel. Next step: stop doing that. * ledger-on-sql: Stop writing in parallel; it causes race conditions. This unfortunately means we also stop _reading_ in parallel, which is less fun. * Revert "ledger-on-sql: Use a sequential log entry ID." This reverts commit c58265bf43b96d16bfa62ed30a795f365b1e83f2. * ledger-on-sql: Create tables with columns that are not nullable. * ledger-on-sql: The H2 conformance tests now work, just slowly. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Run tests against H2 and SQLite on memory and disk. * ledger-on-sql: Allow H2 to read from the log in parallel with writes. * ledger-on-sql: Use UUIDs for entry IDs, not 8 random bytes. * ledger-on-sql: Make sure to log the correlation ID. * ledger-on-sql: Do less while holding a database connection. * ledger-on-sql: Log the connection. * ledger-on-sql: We don't need to tell H2 not to drop the connection. The connection pool takes care of that. * ledger-on-sql: Disable H2 conformance tests on CI; they're too slow. * ledger-on-sql: Rename `loggingContext` to `logCtx`. * ledger-on-sql: Don't abuse log contexts; put the data in the message. * ledger-on-sql: Make the connection log line easier to read. And pull out the logger, even if it's only used once. * ledger-on-sql: Run the unit tests against (H2, SQLite) * (memory, file).
2020-01-22 18:54:51 +03:00
"conformance_test_server_main": "com.daml.ledger.on.sql.MainWithEphemeralDirectory",
"conformance_test_server_args": [
"--jdbc-url=jdbc:sqlite:%DIR/test.sqlite",
],
},
]
ledger-on-sql: Migrate tables using Flyway. (#4232) * ledger-on-sql: Pull out a superclass for the various integration tests. * ledger-on-sql: Use Flyway to migrate tables in an idempotent manner. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Make it easy to run it with Bazel for experimentation. * ledger-on-sql: Test that migrations will never change in the future. * ledger-on-sql: Add a prefix of "ledger_" to the tables. This is so we don't accidentally conflict with the index when the schemas are shared. I am letting myself modify the migrations because the existing migrations haven't been merged into `master` yet. * ledger-on-sql: Explain why we change the pool size after migration. * ledger-api-test-tool: Tests now specify a timeout scale, not a timeout. Makes it easier to change the default timeout in one place. * ledger-api-test-tool: Increase timeouts on slow tests. These tests produce a lot of volume and can make CI flaky. * ledger-on-sql: Only tear down PostgreSQL in tests. For other databases, we just create a new file for each test case. * ledger-on-sql: Reduce the log output in tests. * ledger-on-sql: Use a separate connection pool for Flyway when possible. Apparently `setMaximumPoolSize` doesn't really have the desired effect after the connection pool has already been used. The new test case will be flaky if we process more than one commit in parallel. For SQLite, it seems to be OK. * ledger-on-sql: Use a separate connection pool for Flyway with SQLite. Except in memory. * ledger-on-sql: Use a separate PostgreSQL database for each test. Because performance, innit. Don't have to tear them down. * ledger-on-sql: Move the `Queries` values inside the `RDBMS` objects. * ledger-on-sql: Go into even more detail about pool size hijinks. Co-authored-by: Samir Talwar <samir.talwar@digitalasset.com>
2020-01-28 19:16:23 +03:00
all_database_runtime_deps = {dep: None for db in supported_databases for dep in db["runtime_deps"]}.keys()
da_scala_library(
name = "ledger-on-sql",
srcs = glob(["src/main/scala/**/*.scala"]),
resources = glob(["src/main/resources/**/*"]),
tags = ["maven_coordinates=com.daml.ledger:on-sql:__VERSION__"],
visibility = [
"//visibility:public",
],
runtime_deps = [
"@maven//:ch_qos_logback_logback_classic",
"@maven//:ch_qos_logback_logback_core",
"@maven//:com_h2database_h2",
],
deps = [
"//daml-lf/data",
"//daml-lf/engine",
"//ledger/ledger-api-common",
"//ledger/ledger-api-health",
"//ledger/participant-state",
"//ledger/participant-state/kvutils",
"//ledger/participant-state/kvutils/app",
ledger-on-sql: Get H2 working in the conformance tests. (#4155) * kvutils: Make logback.xml a base file, rather than the only option. * kvutils/app: Simplify logback.base.xml. Mostly by getting rid of unnecessary appenders. * ledger-on-sql: Add trace logging for all database work. * Upgrade H2; there's a few useful bug fixes. * kvutils/app: Let the user override the server JDBC URL. * kvutils/app: Provide a way to specify the ledger ID. * ledger-on-sql: If there are missing entries in the log, fail on read. This can happen right now because we insert in parallel. Next step: stop doing that. * ledger-on-sql: Stop writing in parallel; it causes race conditions. This unfortunately means we also stop _reading_ in parallel, which is less fun. * Revert "ledger-on-sql: Use a sequential log entry ID." This reverts commit c58265bf43b96d16bfa62ed30a795f365b1e83f2. * ledger-on-sql: Create tables with columns that are not nullable. * ledger-on-sql: The H2 conformance tests now work, just slowly. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Run tests against H2 and SQLite on memory and disk. * ledger-on-sql: Allow H2 to read from the log in parallel with writes. * ledger-on-sql: Use UUIDs for entry IDs, not 8 random bytes. * ledger-on-sql: Make sure to log the correlation ID. * ledger-on-sql: Do less while holding a database connection. * ledger-on-sql: Log the connection. * ledger-on-sql: We don't need to tell H2 not to drop the connection. The connection pool takes care of that. * ledger-on-sql: Disable H2 conformance tests on CI; they're too slow. * ledger-on-sql: Rename `loggingContext` to `logCtx`. * ledger-on-sql: Don't abuse log contexts; put the data in the message. * ledger-on-sql: Make the connection log line easier to read. And pull out the logger, even if it's only used once. * ledger-on-sql: Run the unit tests against (H2, SQLite) * (memory, file).
2020-01-22 18:54:51 +03:00
"//libs-scala/contextualized-logging",
"//libs-scala/resources",
"@maven//:com_github_scopt_scopt_2_12",
"@maven//:com_google_protobuf_protobuf_java",
"@maven//:com_typesafe_akka_akka_actor_2_12",
"@maven//:com_typesafe_akka_akka_stream_2_12",
"@maven//:com_typesafe_play_anorm_2_12",
"@maven//:com_typesafe_play_anorm_tokenizer_2_12",
"@maven//:com_zaxxer_HikariCP",
ledger-on-sql: Migrate tables using Flyway. (#4232) * ledger-on-sql: Pull out a superclass for the various integration tests. * ledger-on-sql: Use Flyway to migrate tables in an idempotent manner. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Make it easy to run it with Bazel for experimentation. * ledger-on-sql: Test that migrations will never change in the future. * ledger-on-sql: Add a prefix of "ledger_" to the tables. This is so we don't accidentally conflict with the index when the schemas are shared. I am letting myself modify the migrations because the existing migrations haven't been merged into `master` yet. * ledger-on-sql: Explain why we change the pool size after migration. * ledger-api-test-tool: Tests now specify a timeout scale, not a timeout. Makes it easier to change the default timeout in one place. * ledger-api-test-tool: Increase timeouts on slow tests. These tests produce a lot of volume and can make CI flaky. * ledger-on-sql: Only tear down PostgreSQL in tests. For other databases, we just create a new file for each test case. * ledger-on-sql: Reduce the log output in tests. * ledger-on-sql: Use a separate connection pool for Flyway when possible. Apparently `setMaximumPoolSize` doesn't really have the desired effect after the connection pool has already been used. The new test case will be flaky if we process more than one commit in parallel. For SQLite, it seems to be OK. * ledger-on-sql: Use a separate connection pool for Flyway with SQLite. Except in memory. * ledger-on-sql: Use a separate PostgreSQL database for each test. Because performance, innit. Don't have to tear them down. * ledger-on-sql: Move the `Queries` values inside the `RDBMS` objects. * ledger-on-sql: Go into even more detail about pool size hijinks. Co-authored-by: Samir Talwar <samir.talwar@digitalasset.com>
2020-01-28 19:16:23 +03:00
"@maven//:org_flywaydb_flyway_core",
],
)
da_scala_binary(
name = "app",
main_class = "com.daml.ledger.on.sql.Main",
visibility = ["//visibility:public"],
runtime_deps = all_database_runtime_deps,
deps = [
":ledger-on-sql",
],
)
da_scala_library(
name = "ledger-on-sql-test-lib",
srcs = glob(["src/test/lib/scala/**/*.scala"]),
visibility = [
"//visibility:public",
],
deps = [
":ledger-on-sql",
"//daml-lf/data",
ledger-on-sql: Migrate tables using Flyway. (#4232) * ledger-on-sql: Pull out a superclass for the various integration tests. * ledger-on-sql: Use Flyway to migrate tables in an idempotent manner. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Make it easy to run it with Bazel for experimentation. * ledger-on-sql: Test that migrations will never change in the future. * ledger-on-sql: Add a prefix of "ledger_" to the tables. This is so we don't accidentally conflict with the index when the schemas are shared. I am letting myself modify the migrations because the existing migrations haven't been merged into `master` yet. * ledger-on-sql: Explain why we change the pool size after migration. * ledger-api-test-tool: Tests now specify a timeout scale, not a timeout. Makes it easier to change the default timeout in one place. * ledger-api-test-tool: Increase timeouts on slow tests. These tests produce a lot of volume and can make CI flaky. * ledger-on-sql: Only tear down PostgreSQL in tests. For other databases, we just create a new file for each test case. * ledger-on-sql: Reduce the log output in tests. * ledger-on-sql: Use a separate connection pool for Flyway when possible. Apparently `setMaximumPoolSize` doesn't really have the desired effect after the connection pool has already been used. The new test case will be flaky if we process more than one commit in parallel. For SQLite, it seems to be OK. * ledger-on-sql: Use a separate connection pool for Flyway with SQLite. Except in memory. * ledger-on-sql: Use a separate PostgreSQL database for each test. Because performance, innit. Don't have to tear them down. * ledger-on-sql: Move the `Queries` values inside the `RDBMS` objects. * ledger-on-sql: Go into even more detail about pool size hijinks. Co-authored-by: Samir Talwar <samir.talwar@digitalasset.com>
2020-01-28 19:16:23 +03:00
"//ledger-api/rs-grpc-bridge",
"//ledger-api/testing-utils",
"//ledger/ledger-api-health",
"//ledger/participant-state",
"//ledger/participant-state/kvutils",
ledger-on-sql: Migrate tables using Flyway. (#4232) * ledger-on-sql: Pull out a superclass for the various integration tests. * ledger-on-sql: Use Flyway to migrate tables in an idempotent manner. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Make it easy to run it with Bazel for experimentation. * ledger-on-sql: Test that migrations will never change in the future. * ledger-on-sql: Add a prefix of "ledger_" to the tables. This is so we don't accidentally conflict with the index when the schemas are shared. I am letting myself modify the migrations because the existing migrations haven't been merged into `master` yet. * ledger-on-sql: Explain why we change the pool size after migration. * ledger-api-test-tool: Tests now specify a timeout scale, not a timeout. Makes it easier to change the default timeout in one place. * ledger-api-test-tool: Increase timeouts on slow tests. These tests produce a lot of volume and can make CI flaky. * ledger-on-sql: Only tear down PostgreSQL in tests. For other databases, we just create a new file for each test case. * ledger-on-sql: Reduce the log output in tests. * ledger-on-sql: Use a separate connection pool for Flyway when possible. Apparently `setMaximumPoolSize` doesn't really have the desired effect after the connection pool has already been used. The new test case will be flaky if we process more than one commit in parallel. For SQLite, it seems to be OK. * ledger-on-sql: Use a separate connection pool for Flyway with SQLite. Except in memory. * ledger-on-sql: Use a separate PostgreSQL database for each test. Because performance, innit. Don't have to tear them down. * ledger-on-sql: Move the `Queries` values inside the `RDBMS` objects. * ledger-on-sql: Go into even more detail about pool size hijinks. Co-authored-by: Samir Talwar <samir.talwar@digitalasset.com>
2020-01-28 19:16:23 +03:00
"//ledger/participant-state/kvutils:kvutils-tests-lib",
"//ledger/participant-state/kvutils/app",
ledger-on-sql: Migrate tables using Flyway. (#4232) * ledger-on-sql: Pull out a superclass for the various integration tests. * ledger-on-sql: Use Flyway to migrate tables in an idempotent manner. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Make it easy to run it with Bazel for experimentation. * ledger-on-sql: Test that migrations will never change in the future. * ledger-on-sql: Add a prefix of "ledger_" to the tables. This is so we don't accidentally conflict with the index when the schemas are shared. I am letting myself modify the migrations because the existing migrations haven't been merged into `master` yet. * ledger-on-sql: Explain why we change the pool size after migration. * ledger-api-test-tool: Tests now specify a timeout scale, not a timeout. Makes it easier to change the default timeout in one place. * ledger-api-test-tool: Increase timeouts on slow tests. These tests produce a lot of volume and can make CI flaky. * ledger-on-sql: Only tear down PostgreSQL in tests. For other databases, we just create a new file for each test case. * ledger-on-sql: Reduce the log output in tests. * ledger-on-sql: Use a separate connection pool for Flyway when possible. Apparently `setMaximumPoolSize` doesn't really have the desired effect after the connection pool has already been used. The new test case will be flaky if we process more than one commit in parallel. For SQLite, it seems to be OK. * ledger-on-sql: Use a separate connection pool for Flyway with SQLite. Except in memory. * ledger-on-sql: Use a separate PostgreSQL database for each test. Because performance, innit. Don't have to tear them down. * ledger-on-sql: Move the `Queries` values inside the `RDBMS` objects. * ledger-on-sql: Go into even more detail about pool size hijinks. Co-authored-by: Samir Talwar <samir.talwar@digitalasset.com>
2020-01-28 19:16:23 +03:00
"//libs-scala/contextualized-logging",
"//libs-scala/postgresql-testing",
"//libs-scala/resources",
"@maven//:com_github_scopt_scopt_2_12",
ledger-on-sql: Migrate tables using Flyway. (#4232) * ledger-on-sql: Pull out a superclass for the various integration tests. * ledger-on-sql: Use Flyway to migrate tables in an idempotent manner. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Make it easy to run it with Bazel for experimentation. * ledger-on-sql: Test that migrations will never change in the future. * ledger-on-sql: Add a prefix of "ledger_" to the tables. This is so we don't accidentally conflict with the index when the schemas are shared. I am letting myself modify the migrations because the existing migrations haven't been merged into `master` yet. * ledger-on-sql: Explain why we change the pool size after migration. * ledger-api-test-tool: Tests now specify a timeout scale, not a timeout. Makes it easier to change the default timeout in one place. * ledger-api-test-tool: Increase timeouts on slow tests. These tests produce a lot of volume and can make CI flaky. * ledger-on-sql: Only tear down PostgreSQL in tests. For other databases, we just create a new file for each test case. * ledger-on-sql: Reduce the log output in tests. * ledger-on-sql: Use a separate connection pool for Flyway when possible. Apparently `setMaximumPoolSize` doesn't really have the desired effect after the connection pool has already been used. The new test case will be flaky if we process more than one commit in parallel. For SQLite, it seems to be OK. * ledger-on-sql: Use a separate connection pool for Flyway with SQLite. Except in memory. * ledger-on-sql: Use a separate PostgreSQL database for each test. Because performance, innit. Don't have to tear them down. * ledger-on-sql: Move the `Queries` values inside the `RDBMS` objects. * ledger-on-sql: Go into even more detail about pool size hijinks. Co-authored-by: Samir Talwar <samir.talwar@digitalasset.com>
2020-01-28 19:16:23 +03:00
"@maven//:com_typesafe_akka_akka_actor_2_12",
"@maven//:com_typesafe_akka_akka_stream_2_12",
ledger-on-sql: Migrate tables using Flyway. (#4232) * ledger-on-sql: Pull out a superclass for the various integration tests. * ledger-on-sql: Use Flyway to migrate tables in an idempotent manner. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Make it easy to run it with Bazel for experimentation. * ledger-on-sql: Test that migrations will never change in the future. * ledger-on-sql: Add a prefix of "ledger_" to the tables. This is so we don't accidentally conflict with the index when the schemas are shared. I am letting myself modify the migrations because the existing migrations haven't been merged into `master` yet. * ledger-on-sql: Explain why we change the pool size after migration. * ledger-api-test-tool: Tests now specify a timeout scale, not a timeout. Makes it easier to change the default timeout in one place. * ledger-api-test-tool: Increase timeouts on slow tests. These tests produce a lot of volume and can make CI flaky. * ledger-on-sql: Only tear down PostgreSQL in tests. For other databases, we just create a new file for each test case. * ledger-on-sql: Reduce the log output in tests. * ledger-on-sql: Use a separate connection pool for Flyway when possible. Apparently `setMaximumPoolSize` doesn't really have the desired effect after the connection pool has already been used. The new test case will be flaky if we process more than one commit in parallel. For SQLite, it seems to be OK. * ledger-on-sql: Use a separate connection pool for Flyway with SQLite. Except in memory. * ledger-on-sql: Use a separate PostgreSQL database for each test. Because performance, innit. Don't have to tear them down. * ledger-on-sql: Move the `Queries` values inside the `RDBMS` objects. * ledger-on-sql: Go into even more detail about pool size hijinks. Co-authored-by: Samir Talwar <samir.talwar@digitalasset.com>
2020-01-28 19:16:23 +03:00
"@maven//:org_scalactic_scalactic_2_12",
"@maven//:org_scalatest_scalatest_2_12",
],
)
da_scala_test_suite(
name = "ledger-on-sql-tests",
size = "small",
srcs = glob(["src/test/suite/**/*.scala"]),
data = [
"//ledger/test-common:Test-stable.dar",
],
resources = glob(["src/test/resources/*"]),
ledger-on-sql: Migrate tables using Flyway. (#4232) * ledger-on-sql: Pull out a superclass for the various integration tests. * ledger-on-sql: Use Flyway to migrate tables in an idempotent manner. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Make it easy to run it with Bazel for experimentation. * ledger-on-sql: Test that migrations will never change in the future. * ledger-on-sql: Add a prefix of "ledger_" to the tables. This is so we don't accidentally conflict with the index when the schemas are shared. I am letting myself modify the migrations because the existing migrations haven't been merged into `master` yet. * ledger-on-sql: Explain why we change the pool size after migration. * ledger-api-test-tool: Tests now specify a timeout scale, not a timeout. Makes it easier to change the default timeout in one place. * ledger-api-test-tool: Increase timeouts on slow tests. These tests produce a lot of volume and can make CI flaky. * ledger-on-sql: Only tear down PostgreSQL in tests. For other databases, we just create a new file for each test case. * ledger-on-sql: Reduce the log output in tests. * ledger-on-sql: Use a separate connection pool for Flyway when possible. Apparently `setMaximumPoolSize` doesn't really have the desired effect after the connection pool has already been used. The new test case will be flaky if we process more than one commit in parallel. For SQLite, it seems to be OK. * ledger-on-sql: Use a separate connection pool for Flyway with SQLite. Except in memory. * ledger-on-sql: Use a separate PostgreSQL database for each test. Because performance, innit. Don't have to tear them down. * ledger-on-sql: Move the `Queries` values inside the `RDBMS` objects. * ledger-on-sql: Go into even more detail about pool size hijinks. Co-authored-by: Samir Talwar <samir.talwar@digitalasset.com>
2020-01-28 19:16:23 +03:00
runtime_deps = all_database_runtime_deps,
deps = [
":ledger-on-sql",
ledger-on-sql: Migrate tables using Flyway. (#4232) * ledger-on-sql: Pull out a superclass for the various integration tests. * ledger-on-sql: Use Flyway to migrate tables in an idempotent manner. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Make it easy to run it with Bazel for experimentation. * ledger-on-sql: Test that migrations will never change in the future. * ledger-on-sql: Add a prefix of "ledger_" to the tables. This is so we don't accidentally conflict with the index when the schemas are shared. I am letting myself modify the migrations because the existing migrations haven't been merged into `master` yet. * ledger-on-sql: Explain why we change the pool size after migration. * ledger-api-test-tool: Tests now specify a timeout scale, not a timeout. Makes it easier to change the default timeout in one place. * ledger-api-test-tool: Increase timeouts on slow tests. These tests produce a lot of volume and can make CI flaky. * ledger-on-sql: Only tear down PostgreSQL in tests. For other databases, we just create a new file for each test case. * ledger-on-sql: Reduce the log output in tests. * ledger-on-sql: Use a separate connection pool for Flyway when possible. Apparently `setMaximumPoolSize` doesn't really have the desired effect after the connection pool has already been used. The new test case will be flaky if we process more than one commit in parallel. For SQLite, it seems to be OK. * ledger-on-sql: Use a separate connection pool for Flyway with SQLite. Except in memory. * ledger-on-sql: Use a separate PostgreSQL database for each test. Because performance, innit. Don't have to tear them down. * ledger-on-sql: Move the `Queries` values inside the `RDBMS` objects. * ledger-on-sql: Go into even more detail about pool size hijinks. Co-authored-by: Samir Talwar <samir.talwar@digitalasset.com>
2020-01-28 19:16:23 +03:00
":ledger-on-sql-test-lib",
"//daml-lf/data",
"//ledger-api/rs-grpc-bridge",
"//ledger-api/testing-utils",
"//ledger/ledger-api-common",
"//ledger/ledger-api-health",
"//ledger/participant-state",
"//ledger/participant-state/kvutils",
"//ledger/participant-state/kvutils:kvutils-tests-lib",
"//libs-scala/contextualized-logging",
"//libs-scala/postgresql-testing",
"//libs-scala/resources",
"@maven//:com_typesafe_akka_akka_actor_2_12",
"@maven//:com_typesafe_akka_akka_stream_2_12",
ledger-on-sql: Migrate tables using Flyway. (#4232) * ledger-on-sql: Pull out a superclass for the various integration tests. * ledger-on-sql: Use Flyway to migrate tables in an idempotent manner. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Make it easy to run it with Bazel for experimentation. * ledger-on-sql: Test that migrations will never change in the future. * ledger-on-sql: Add a prefix of "ledger_" to the tables. This is so we don't accidentally conflict with the index when the schemas are shared. I am letting myself modify the migrations because the existing migrations haven't been merged into `master` yet. * ledger-on-sql: Explain why we change the pool size after migration. * ledger-api-test-tool: Tests now specify a timeout scale, not a timeout. Makes it easier to change the default timeout in one place. * ledger-api-test-tool: Increase timeouts on slow tests. These tests produce a lot of volume and can make CI flaky. * ledger-on-sql: Only tear down PostgreSQL in tests. For other databases, we just create a new file for each test case. * ledger-on-sql: Reduce the log output in tests. * ledger-on-sql: Use a separate connection pool for Flyway when possible. Apparently `setMaximumPoolSize` doesn't really have the desired effect after the connection pool has already been used. The new test case will be flaky if we process more than one commit in parallel. For SQLite, it seems to be OK. * ledger-on-sql: Use a separate connection pool for Flyway with SQLite. Except in memory. * ledger-on-sql: Use a separate PostgreSQL database for each test. Because performance, innit. Don't have to tear them down. * ledger-on-sql: Move the `Queries` values inside the `RDBMS` objects. * ledger-on-sql: Go into even more detail about pool size hijinks. Co-authored-by: Samir Talwar <samir.talwar@digitalasset.com>
2020-01-28 19:16:23 +03:00
"@maven//:org_flywaydb_flyway_core",
"@maven//:org_scala_lang_modules_scala_java8_compat_2_12",
"@maven//:org_scalactic_scalactic_2_12",
"@maven//:org_scalatest_scalatest_2_12",
],
)
[
(
da_scala_binary(
name = "conformance-test-{}-bin".format(db["name"]),
main_class = db.get("conformance_test_server_main", "com.daml.ledger.on.sql.Main"),
visibility = ["//visibility:public"],
runtime_deps = db.get("runtime_deps", []),
deps = [
":ledger-on-sql",
":ledger-on-sql-test-lib",
],
),
conformance_test(
name = "conformance-test-{}".format(db["name"]),
ports = [6865],
server = ":conformance-test-{}-bin".format(db["name"]),
server_args = ["--port=6865"] + db.get("conformance_test_server_args", []),
tags = db.get("conformance_test_tags", []),
ledger-on-sql: Get H2 working in the conformance tests. (#4155) * kvutils: Make logback.xml a base file, rather than the only option. * kvutils/app: Simplify logback.base.xml. Mostly by getting rid of unnecessary appenders. * ledger-on-sql: Add trace logging for all database work. * Upgrade H2; there's a few useful bug fixes. * kvutils/app: Let the user override the server JDBC URL. * kvutils/app: Provide a way to specify the ledger ID. * ledger-on-sql: If there are missing entries in the log, fail on read. This can happen right now because we insert in parallel. Next step: stop doing that. * ledger-on-sql: Stop writing in parallel; it causes race conditions. This unfortunately means we also stop _reading_ in parallel, which is less fun. * Revert "ledger-on-sql: Use a sequential log entry ID." This reverts commit c58265bf43b96d16bfa62ed30a795f365b1e83f2. * ledger-on-sql: Create tables with columns that are not nullable. * ledger-on-sql: The H2 conformance tests now work, just slowly. CHANGELOG_BEGIN CHANGELOG_END * ledger-on-sql: Run tests against H2 and SQLite on memory and disk. * ledger-on-sql: Allow H2 to read from the log in parallel with writes. * ledger-on-sql: Use UUIDs for entry IDs, not 8 random bytes. * ledger-on-sql: Make sure to log the correlation ID. * ledger-on-sql: Do less while holding a database connection. * ledger-on-sql: Log the connection. * ledger-on-sql: We don't need to tell H2 not to drop the connection. The connection pool takes care of that. * ledger-on-sql: Disable H2 conformance tests on CI; they're too slow. * ledger-on-sql: Rename `loggingContext` to `logCtx`. * ledger-on-sql: Don't abuse log contexts; put the data in the message. * ledger-on-sql: Make the connection log line easier to read. And pull out the logger, even if it's only used once. * ledger-on-sql: Run the unit tests against (H2, SQLite) * (memory, file).
2020-01-22 18:54:51 +03:00
test_tool_args = db.get("conformance_test_tool_args", []) + [
"--all-tests",
"--exclude=ConfigManagementServiceIT",
"--exclude=TimeIT",
],
),
conformance_test(
name = "conformance-test-config-management-{}".format(db["name"]),
ports = [6865],
server = ":conformance-test-{}-bin".format(db["name"]),
server_args = ["--port=6865"] + db.get("conformance_test_server_args", []),
tags = db.get("conformance_test_tags", []),
test_tool_args = db.get("conformance_test_tool_args", []) + [
"--include=ConfigManagementServiceIT",
],
),
)
for db in supported_databases
]