2020-03-31 13:56:11 +03:00
|
|
|
[package]
|
|
|
|
|
name = "phases"
|
|
|
|
|
edition = "2018"
|
|
|
|
|
version = "0.1.0"
|
|
|
|
|
authors = ['Facebook']
|
|
|
|
|
license = "GPLv2+"
|
|
|
|
|
autotests = false
|
|
|
|
|
include = ["schemas/**/*.sql", "src/**/*.rs", "tests/src/**/*.rs"]
|
|
|
|
|
|
|
|
|
|
[lib]
|
|
|
|
|
path = "src/lib.rs"
|
|
|
|
|
|
|
|
|
|
[[test]]
|
|
|
|
|
name = "tests"
|
|
|
|
|
path = "tests/src/main.rs"
|
|
|
|
|
|
|
|
|
|
[dependencies]
|
|
|
|
|
caching_ext = { path = "../common/rust/caching_ext" }
|
|
|
|
|
changeset_fetcher = { path = "../blobrepo/changeset_fetcher" }
|
|
|
|
|
context = { path = "../server/context" }
|
|
|
|
|
mononoke_types = { path = "../mononoke_types" }
|
migrate from sql_ext::SqlConstructors to sql_construct
Summary:
Migrate the configuration of sql data managers from the old configuration using `sql_ext::SqlConstructors` to the new configuration using `sql_construct::SqlConstruct`.
In the old configuration, sharded filenodes were included in the configuration of remote databases, even when that made no sense:
```
[storage.db.remote]
db_address = "main_database"
sharded_filenodes = { shard_map = "sharded_database", shard_num = 100 }
[storage.blobstore.multiplexed]
queue_db = { remote = {
db_address = "queue_database",
sharded_filenodes = { shard_map = "valid_config_but_meaningless", shard_num = 100 }
}
```
This change separates out:
* **DatabaseConfig**, which describes a single local or remote connection to a database, used in configuration like the queue database.
* **MetadataDatabaseConfig**, which describes the multiple databases used for repo metadata.
**MetadataDatabaseConfig** is either:
* **Local**, which is a local sqlite database, the same as for **DatabaseConfig**; or
* **Remote**, which contains:
* `primary`, the database used for main metadata.
* `filenodes`, the database used for filenodes, which may be sharded or unsharded.
More fields can be added to **RemoteMetadataDatabaseConfig** when we want to add new databases.
New configuration looks like:
```
[storage.metadata.remote]
primary = { db_address = "main_database" }
filenodes = { sharded = { shard_map = "sharded_database", shard_num = 100 } }
[storage.blobstore.multiplexed]
queue_db = { remote = { db_address = "queue_database" } }
```
The `sql_construct` crate facilitates this by providing the following traits:
* **SqlConstruct** defines the basic rules for construction, and allows construction based on a local sqlite database.
* **SqlShardedConstruct** defines the basic rules for construction based on sharded databases.
* **FbSqlConstruct** and **FbShardedSqlConstruct** allow construction based on unsharded and sharded remote databases on Facebook infra.
* **SqlConstructFromDatabaseConfig** allows construction based on the database defined in **DatabaseConfig**.
* **SqlConstructFromMetadataDatabaseConfig** allows construction based on the appropriate database defined in **MetadataDatabaseConfig**.
* **SqlShardableConstructFromMetadataDatabaseConfig** allows construction based on the appropriate shardable databases defined in **MetadataDatabaseConfig**.
Sql database managers should implement:
* **SqlConstruct** in order to define how to construct an unsharded instance from a single set of `SqlConnections`.
* **SqlShardedConstruct**, if they are shardable, in order to define how to construct a sharded instance.
* If the database is part of the repository metadata database config, either of:
* **SqlConstructFromMetadataDatabaseConfig** if they are not shardable. By default they will use the primary metadata database, but this can be overridden by implementing `remote_database_config`.
* **SqlShardableConstructFromMetadataDatabaseConfig** if they are shardable. They must implement `remote_database_config` to specify where to get the sharded or unsharded configuration from.
Reviewed By: StanislavGlebik
Differential Revision: D20734883
fbshipit-source-id: bb2f4cb3806edad2bbd54a47558a164e3190c5d1
2020-04-02 15:24:53 +03:00
|
|
|
sql_construct = { path = "../common/sql_construct" }
|
2020-03-31 13:56:11 +03:00
|
|
|
sql_ext = { path = "../common/rust/sql_ext" }
|
|
|
|
|
cachelib = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
|
cloned = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
|
fbinit = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
|
futures_ext = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
|
memcache = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
|
sql = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
|
stats = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
|
abomonation = "0.7"
|
|
|
|
|
abomonation_derive = "0.5"
|
|
|
|
|
anyhow = "1.0"
|
|
|
|
|
ascii = "1.0"
|
|
|
|
|
bytes = { version = "0.5", features = ["serde"] }
|
|
|
|
|
futures = { version = "0.3", features = ["async-await", "compat"] }
|
|
|
|
|
futures-old = { package = "futures", version = "0.1" }
|
|
|
|
|
thiserror = "1.0"
|
|
|
|
|
|
|
|
|
|
[dev-dependencies]
|
|
|
|
|
blobrepo = { path = "../blobrepo" }
|
2020-06-22 17:20:47 +03:00
|
|
|
blobrepo_hg = { path = "../blobrepo/blobrepo_hg" }
|
2020-03-31 13:56:11 +03:00
|
|
|
bookmarks = { path = "../bookmarks" }
|
|
|
|
|
fixtures = { path = "../tests/fixtures" }
|
2020-06-22 23:48:38 +03:00
|
|
|
mercurial_types = { path = "../mercurial/types" }
|
2020-03-31 13:56:11 +03:00
|
|
|
mononoke_types-mocks = { path = "../mononoke_types/mocks" }
|
|
|
|
|
maplit = "1.0"
|
2020-04-14 13:35:41 +03:00
|
|
|
tokio-compat = "0.1"
|
