2020-06-17 16:26:05 +03:00
|
|
|
[package]
|
|
|
|
name = "eden_mononoke"
|
|
|
|
version = "0.1.0"
|
2021-02-19 22:02:06 +03:00
|
|
|
authors = ["Facebook"]
|
|
|
|
edition = "2018"
|
2020-06-17 16:26:05 +03:00
|
|
|
license = "GPLv2+"
|
2020-07-11 08:08:09 +03:00
|
|
|
|
2020-06-17 16:26:05 +03:00
|
|
|
[[bin]]
|
2021-02-17 23:40:17 +03:00
|
|
|
name = "backfill_mapping"
|
|
|
|
path = "cmds/backfill_mapping.rs"
|
2020-06-17 16:26:05 +03:00
|
|
|
|
|
|
|
[[bin]]
|
|
|
|
name = "benchmark_storage_config"
|
|
|
|
path = "cmds/benchmark_storage_config/main.rs"
|
|
|
|
|
|
|
|
[[bin]]
|
|
|
|
name = "bonsai_verify"
|
|
|
|
path = "cmds/bonsai_verify/main.rs"
|
2021-03-02 17:41:22 +03:00
|
|
|
test = false
|
2020-06-17 16:26:05 +03:00
|
|
|
|
|
|
|
[[bin]]
|
|
|
|
name = "configlint"
|
|
|
|
path = "cmds/configlint.rs"
|
|
|
|
|
|
|
|
[[bin]]
|
|
|
|
name = "dumprev"
|
|
|
|
path = "cmds/dumprev.rs"
|
2021-03-02 17:41:22 +03:00
|
|
|
test = false
|
2020-06-17 16:26:05 +03:00
|
|
|
|
|
|
|
[[bin]]
|
|
|
|
name = "idxdump"
|
|
|
|
path = "cmds/idxdump.rs"
|
2021-03-02 17:41:22 +03:00
|
|
|
test = false
|
2020-06-17 16:26:05 +03:00
|
|
|
|
2020-07-10 10:59:29 +03:00
|
|
|
[[bin]]
|
|
|
|
name = "manual_scrub"
|
|
|
|
path = "cmds/manual_scrub/main.rs"
|
|
|
|
|
2021-04-20 22:44:12 +03:00
|
|
|
[[bin]]
|
|
|
|
name = "packer"
|
|
|
|
path = "cmds/packer/main.rs"
|
|
|
|
|
2020-06-17 16:26:05 +03:00
|
|
|
[[bin]]
|
|
|
|
name = "rechunker"
|
|
|
|
path = "cmds/rechunker.rs"
|
|
|
|
|
|
|
|
[[bin]]
|
|
|
|
name = "revlogrepo"
|
|
|
|
path = "cmds/revlogrepo.rs"
|
|
|
|
|
2020-10-08 19:41:16 +03:00
|
|
|
[[bin]]
|
|
|
|
name = "segmented_changelog_seeder"
|
|
|
|
path = "cmds/segmented_changelog_seeder.rs"
|
|
|
|
|
2020-10-28 20:22:18 +03:00
|
|
|
[[bin]]
|
2020-10-30 03:35:41 +03:00
|
|
|
name = "segmented_changelog_tailer"
|
|
|
|
path = "cmds/segmented_changelog_tailer.rs"
|
|
|
|
|
|
|
|
[[bin]]
|
2020-10-28 20:22:18 +03:00
|
|
|
name = "sqlblob_gc"
|
|
|
|
path = "cmds/sqlblob_gc/main.rs"
|
|
|
|
|
2020-09-01 17:12:10 +03:00
|
|
|
[[bin]]
|
|
|
|
name = "streaming_clone_warmup"
|
|
|
|
path = "cmds/streaming_clone_warmup/main.rs"
|
|
|
|
|
2020-06-17 16:26:05 +03:00
|
|
|
[dependencies]
|
2021-02-19 22:02:06 +03:00
|
|
|
anyhow = "1.0"
|
|
|
|
ascii = "1.0"
|
2021-05-05 22:19:32 +03:00
|
|
|
async-compression = { version = "0.3.8", features = ["all"] }
|
2021-02-26 02:09:13 +03:00
|
|
|
blobrepo = { version = "0.1.0", path = "blobrepo" }
|
|
|
|
blobrepo_hg = { version = "0.1.0", path = "blobrepo/blobrepo_hg" }
|
|
|
|
blobrepo_utils = { version = "0.1.0", path = "blobrepo_utils" }
|
|
|
|
blobstore = { version = "0.1.0", path = "blobstore" }
|
|
|
|
blobstore_factory = { version = "0.1.0", path = "blobstore/factory" }
|
|
|
|
bookmarks = { version = "0.1.0", path = "bookmarks" }
|
2021-02-19 22:02:06 +03:00
|
|
|
bytesize = "1.0"
|
2021-02-26 02:09:13 +03:00
|
|
|
cacheblob = { version = "0.1.0", path = "blobstore/cacheblob" }
|
|
|
|
cached_config = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
2021-02-19 22:02:06 +03:00
|
|
|
clap = "2.33"
|
2021-02-26 02:09:13 +03:00
|
|
|
cloned = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
cmdlib = { version = "0.1.0", path = "cmdlib" }
|
|
|
|
context = { version = "0.1.0", path = "server/context" }
|
2021-02-19 22:02:06 +03:00
|
|
|
criterion = "=0.3.1"
|
2021-04-06 18:42:33 +03:00
|
|
|
derive_more = "0.99.3"
|
mononoke: use MononokeEnvironment in RepoFactory
Summary:
There is a very frustrating operation that happens often when working on the
Mononoke code base:
- You want to add a flag
- You want to consume it in the repo somewhere
Unfortunately, when we need to do this, we end up having to thread this from a
million places and parse it out in every single main() we have.
This is a mess, and it results in every single Mononoke binary starting with
heaps of useless boilerplate:
```
let matches = app.get_matches();
let (caching, logger, mut runtime) = matches.init_mononoke(fb)?;
let config_store = args::init_config_store(fb, &logger, &matches)?;
let mysql_options = args::parse_mysql_options(&matches);
let blobstore_options = args::parse_blobstore_options(&matches)?;
let readonly_storage = args::parse_readonly_storage(&matches);
```
So, this diff updates us to just use MononokeEnvironment directly in
RepoFactory, which means none of that has to happen: we can now add a flag,
parse it into MononokeEnvironment, and get going.
While we're at it, we can also remove blobstore options and all that jazz from
MononokeApiEnvironment since now it's there in the underlying RepoFactory.
Reviewed By: HarveyHunt
Differential Revision: D27767700
fbshipit-source-id: e1e359bf403b4d3d7b36e5f670aa1a7dd4f1d209
2021-04-16 20:26:03 +03:00
|
|
|
environment = { version = "0.1.0", path = "cmdlib/environment" }
|
2021-02-26 02:09:13 +03:00
|
|
|
failure_ext = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
fbinit = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
filestore = { version = "0.1.0", path = "filestore" }
|
2021-03-04 17:40:30 +03:00
|
|
|
futures = { version = "0.3.13", features = ["async-await", "compat"] }
|
|
|
|
futures-old = { package = "futures", version = "0.1.31" }
|
2021-02-19 22:02:06 +03:00
|
|
|
futures-util = "0.3.7"
|
|
|
|
itertools = "0.8"
|
2021-02-26 02:09:13 +03:00
|
|
|
lock_ext = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
mercurial_derived_data = { version = "0.1.0", path = "derived_data/mercurial_derived_data" }
|
|
|
|
mercurial_revlog = { version = "0.1.0", path = "mercurial/revlog" }
|
|
|
|
mercurial_types = { version = "0.1.0", path = "mercurial/types" }
|
|
|
|
metaconfig_types = { version = "0.1.0", path = "metaconfig/types" }
|
|
|
|
mononoke_types = { version = "0.1.0", path = "mononoke_types" }
|
2021-04-20 22:44:12 +03:00
|
|
|
packblob = { version = "0.1.0", path = "blobstore/packblob" }
|
2021-02-26 02:09:13 +03:00
|
|
|
prefixblob = { version = "0.1.0", path = "blobstore/prefixblob" }
|
2021-02-19 22:02:06 +03:00
|
|
|
rand = { version = "0.7", features = ["small_rng"] }
|
2021-04-07 23:59:27 +03:00
|
|
|
repo_factory = { version = "0.1.0", path = "repo_factory" }
|
2021-02-26 02:09:13 +03:00
|
|
|
revset = { version = "0.1.0", path = "revset" }
|
|
|
|
scuba_ext = { version = "0.1.0", path = "common/scuba_ext" }
|
|
|
|
segmented_changelog = { version = "0.1.0", path = "segmented_changelog" }
|
2021-02-23 05:14:45 +03:00
|
|
|
serde = { version = "=1.0.118", features = ["derive", "rc"] }
|
2021-02-19 22:02:06 +03:00
|
|
|
serde_derive = "1.0"
|
2021-05-03 20:29:23 +03:00
|
|
|
slog = { version = "2.5", features = ["max_level_trace"] }
|
2021-02-26 02:09:13 +03:00
|
|
|
sql_construct = { version = "0.1.0", path = "common/sql_construct" }
|
|
|
|
sql_ext = { version = "0.1.0", path = "common/rust/sql_ext" }
|
|
|
|
sqlblob = { version = "0.1.0", path = "blobstore/sqlblob" }
|
|
|
|
streaming_clone = { version = "0.1.0", path = "repo_client/streaming_clone" }
|
2021-04-07 19:29:57 +03:00
|
|
|
tempfile = "3.1"
|
mononoke: update to tokio 1.x
Summary:
NOTE: there is one final pre-requisite here, which is that we should default all Mononoke binaries to `--use-mysql-client` because the other SQL client implementations will break once this lands. That said, this is probably the right time to start reviewing.
There's a lot going on here, but Tokio updates being what they are, it has to happen as just one diff (though I did try to minimize churn by modernizing a bunch of stuff in earlier diffs).
Here's a detailed list of what is going on:
- I had to add a number `cargo_toml_dir` for binaries in `eden/mononoke/TARGETS`, because we have to use 2 versions of Bytes concurrently at this time, and the two cannot co-exist in the same Cargo workspace.
- Lots of little Tokio changes:
- Stream abstractions moving to `tokio-stream`
- `tokio::time::delay_for` became `tokio::time::sleep`
- `tokio::sync::watch::Sender::send` became `tokio::sync::watch::Sender::broadcast`
- `tokio::sync::Semaphore::acquire` returns a `Result` now.
- `tokio::runtime::Runtime::block_on` no longer takes a `&mut self` (just a `&self`).
- `Notify` grew a few more methods with different semantics. We only use this in tests, I used what seemed logical given the use case.
- Runtime builders have changed quite a bit:
- My `no_coop` patch is gone in Tokio 1.x, but it has a new `tokio::task::unconstrained` wrapper (also from me), which I included on `MononokeApi::new`.
- Tokio now detects your logical CPUs, not physical CPUs, so we no longer need to use `num_cpus::get()` to figure it out.
- Tokio 1.x now uses Bytes 1.x:
- At the edges (i.e. streams returned to Hyper or emitted by RepoClient), we need to return Bytes 1.x. However, internally we still use Bytes 0.5 in some places (notably: Filestore).
- In LFS, this means we make a copy. We used to do that a while ago anyway (in the other direction) and it was never a meaningful CPU cost, so I think this is fine.
- In Mononoke Server it doesn't really matter because that still generates ... Bytes 0.1 anyway so there was a copy before from 0.1 to 0.5 and it's from 0.1 to 1.x.
- In the very few places where we read stuff using Tokio from the outside world (historical import tools for LFS), we copy.
- tokio-tls changed a lot, they removed all the convenience methods around connecting. This resulted in updates to:
- How we listen in Mononoke Server & LFS
- How we connect in hgcli.
- Note: all this stuff has test coverage.
- The child process API changed a little bit. We used to have a ChildWrapper around the hg sync job to make a Tokio 0.2.x child look more like a Tokio 1.x Child, so now we can just remove this.
- Hyper changed their Websocket upgrade mechanism (you now need the whole `Request` to upgrade, whereas before that you needed just the `Body`, so I changed up our code a little bit in Mononoke's HTTP acceptor to defer splitting up the `Request` into parts until after we know whether we plan to upgrade it.
- I removed the MySQL tests that didn't use mysql client, because we're leaving that behind and don't intend to support it on Tokio 1.x.
Reviewed By: mitrandir77
Differential Revision: D26669620
fbshipit-source-id: acb6aff92e7f70a7a43f32cf758f252f330e60c9
2021-04-28 17:35:21 +03:00
|
|
|
tokio = { version = "1.4", features = ["full", "test-util"] }
|
|
|
|
tokio-stream = { version = "0.1.4", features = ["fs", "io-util", "net", "sync", "time"] }
|
2020-12-17 18:43:34 +03:00
|
|
|
toml = "=0.5.7"
|
2020-06-17 16:26:05 +03:00
|
|
|
|
|
|
|
[patch.crates-io]
|
2021-02-26 02:09:13 +03:00
|
|
|
addr2line = { git = "https://github.com/gimli-rs/addr2line.git", rev = "0b6b6018b5b252a18e628fba03885f7d21844b3c" }
|
2020-11-19 23:56:30 +03:00
|
|
|
bytecount = { git = "https://github.com/llogiq/bytecount", rev = "469eaf8395c99397cd64d059737a9054aa014088" }
|
2020-06-17 16:26:05 +03:00
|
|
|
chashmap = { git = "https://gitlab.redox-os.org/ahornby/chashmap", rev = "901ace2ca3cdbc2095adb1af111d211e254e2aae" }
|
|
|
|
const-random = { git = "https://github.com/fbsource/const-random", rev = "374c5b46427fe2ffbf6acbd9c1687e0f1a809f95" }
|
2021-04-08 21:48:52 +03:00
|
|
|
curl = { git = "https://github.com/kulshrax/curl-rust", rev = "2a15bbd8dbbd54734313fa703a64db7ce6ddaff0" }
|
|
|
|
curl-sys = { git = "https://github.com/kulshrax/curl-rust", rev = "2a15bbd8dbbd54734313fa703a64db7ce6ddaff0" }
|
2021-02-26 02:09:13 +03:00
|
|
|
enumset = { git = "https://github.com/danobi/enumset", rev = "4c01c583c27a725948fededbfb3461c572a669a4" }
|
2020-11-21 03:58:47 +03:00
|
|
|
lru-disk-cache = { git = "https://github.com/mozilla/sccache", rev = "033ebaae69beeb0ac04e8c35d6ff1103487bd9a3" }
|
2021-05-05 20:31:41 +03:00
|
|
|
mysql_common = { git = "https://github.com/iammxt/rust_mysql_common", rev = "0e4c86952f1e799960e736c0b2bb9d2a6d935bf1" }
|
2021-03-03 15:06:49 +03:00
|
|
|
openssl = { git = "https://github.com/sfackler/rust-openssl", rev = "68fc8ba890d77986b06ca5ce29d1089285fbbcf9" }
|
|
|
|
openssl-sys = { git = "https://github.com/sfackler/rust-openssl", rev = "68fc8ba890d77986b06ca5ce29d1089285fbbcf9" }
|
2021-03-31 12:32:58 +03:00
|
|
|
petgraph = { git = "https://github.com/jkeljo/petgraph", rev = "e3e9dd8632d23973fdc0b42c1117d5e5fc5fa384" }
|
2020-06-17 16:26:05 +03:00
|
|
|
prost = { git = "https://github.com/gabrielrussoc/prost", branch = "protoc-runtime" }
|
|
|
|
prost-build = { git = "https://github.com/gabrielrussoc/prost", branch = "protoc-runtime" }
|
|
|
|
prost-derive = { git = "https://github.com/gabrielrussoc/prost", branch = "protoc-runtime" }
|
|
|
|
prost-types = { git = "https://github.com/gabrielrussoc/prost", branch = "protoc-runtime" }
|
|
|
|
r2d2_sqlite = { git = "https://github.com/jsgf/r2d2-sqlite.git", rev = "6d77a828ca0a3c507a3f58561532a1b6c66c7918" }
|
|
|
|
rustfilt = { git = "https://github.com/jsgf/rustfilt.git", rev = "8141fa7f1caee562ee8daffb2ddeca3d1f0d36e5" }
|
|
|
|
tokio-core = { git = "https://github.com/bolinfest/tokio-core", rev = "5f37aa3c627d56ee49154bc851d6930f5ab4398f" }
|
2020-12-17 18:43:34 +03:00
|
|
|
toml = { git = "https://github.com/jsgf/toml-rs", branch = "dotted-table-0.5.7" }
|
2021-02-26 02:09:13 +03:00
|
|
|
|
|
|
|
[workspace]
|
|
|
|
members = [
|
|
|
|
".",
|
mononoke: update to tokio 1.x
Summary:
NOTE: there is one final pre-requisite here, which is that we should default all Mononoke binaries to `--use-mysql-client` because the other SQL client implementations will break once this lands. That said, this is probably the right time to start reviewing.
There's a lot going on here, but Tokio updates being what they are, it has to happen as just one diff (though I did try to minimize churn by modernizing a bunch of stuff in earlier diffs).
Here's a detailed list of what is going on:
- I had to add a number `cargo_toml_dir` for binaries in `eden/mononoke/TARGETS`, because we have to use 2 versions of Bytes concurrently at this time, and the two cannot co-exist in the same Cargo workspace.
- Lots of little Tokio changes:
- Stream abstractions moving to `tokio-stream`
- `tokio::time::delay_for` became `tokio::time::sleep`
- `tokio::sync::watch::Sender::send` became `tokio::sync::watch::Sender::broadcast`
- `tokio::sync::Semaphore::acquire` returns a `Result` now.
- `tokio::runtime::Runtime::block_on` no longer takes a `&mut self` (just a `&self`).
- `Notify` grew a few more methods with different semantics. We only use this in tests, I used what seemed logical given the use case.
- Runtime builders have changed quite a bit:
- My `no_coop` patch is gone in Tokio 1.x, but it has a new `tokio::task::unconstrained` wrapper (also from me), which I included on `MononokeApi::new`.
- Tokio now detects your logical CPUs, not physical CPUs, so we no longer need to use `num_cpus::get()` to figure it out.
- Tokio 1.x now uses Bytes 1.x:
- At the edges (i.e. streams returned to Hyper or emitted by RepoClient), we need to return Bytes 1.x. However, internally we still use Bytes 0.5 in some places (notably: Filestore).
- In LFS, this means we make a copy. We used to do that a while ago anyway (in the other direction) and it was never a meaningful CPU cost, so I think this is fine.
- In Mononoke Server it doesn't really matter because that still generates ... Bytes 0.1 anyway so there was a copy before from 0.1 to 0.5 and it's from 0.1 to 1.x.
- In the very few places where we read stuff using Tokio from the outside world (historical import tools for LFS), we copy.
- tokio-tls changed a lot, they removed all the convenience methods around connecting. This resulted in updates to:
- How we listen in Mononoke Server & LFS
- How we connect in hgcli.
- Note: all this stuff has test coverage.
- The child process API changed a little bit. We used to have a ChildWrapper around the hg sync job to make a Tokio 0.2.x child look more like a Tokio 1.x Child, so now we can just remove this.
- Hyper changed their Websocket upgrade mechanism (you now need the whole `Request` to upgrade, whereas before that you needed just the `Body`, so I changed up our code a little bit in Mononoke's HTTP acceptor to defer splitting up the `Request` into parts until after we know whether we plan to upgrade it.
- I removed the MySQL tests that didn't use mysql client, because we're leaving that behind and don't intend to support it on Tokio 1.x.
Reviewed By: mitrandir77
Differential Revision: D26669620
fbshipit-source-id: acb6aff92e7f70a7a43f32cf758f252f330e60c9
2021-04-28 17:35:21 +03:00
|
|
|
"admin",
|
|
|
|
"aliasverify",
|
2021-02-26 02:09:13 +03:00
|
|
|
"alpn",
|
mononoke: update to tokio 1.x
Summary:
NOTE: there is one final pre-requisite here, which is that we should default all Mononoke binaries to `--use-mysql-client` because the other SQL client implementations will break once this lands. That said, this is probably the right time to start reviewing.
There's a lot going on here, but Tokio updates being what they are, it has to happen as just one diff (though I did try to minimize churn by modernizing a bunch of stuff in earlier diffs).
Here's a detailed list of what is going on:
- I had to add a number `cargo_toml_dir` for binaries in `eden/mononoke/TARGETS`, because we have to use 2 versions of Bytes concurrently at this time, and the two cannot co-exist in the same Cargo workspace.
- Lots of little Tokio changes:
- Stream abstractions moving to `tokio-stream`
- `tokio::time::delay_for` became `tokio::time::sleep`
- `tokio::sync::watch::Sender::send` became `tokio::sync::watch::Sender::broadcast`
- `tokio::sync::Semaphore::acquire` returns a `Result` now.
- `tokio::runtime::Runtime::block_on` no longer takes a `&mut self` (just a `&self`).
- `Notify` grew a few more methods with different semantics. We only use this in tests, I used what seemed logical given the use case.
- Runtime builders have changed quite a bit:
- My `no_coop` patch is gone in Tokio 1.x, but it has a new `tokio::task::unconstrained` wrapper (also from me), which I included on `MononokeApi::new`.
- Tokio now detects your logical CPUs, not physical CPUs, so we no longer need to use `num_cpus::get()` to figure it out.
- Tokio 1.x now uses Bytes 1.x:
- At the edges (i.e. streams returned to Hyper or emitted by RepoClient), we need to return Bytes 1.x. However, internally we still use Bytes 0.5 in some places (notably: Filestore).
- In LFS, this means we make a copy. We used to do that a while ago anyway (in the other direction) and it was never a meaningful CPU cost, so I think this is fine.
- In Mononoke Server it doesn't really matter because that still generates ... Bytes 0.1 anyway so there was a copy before from 0.1 to 0.5 and it's from 0.1 to 1.x.
- In the very few places where we read stuff using Tokio from the outside world (historical import tools for LFS), we copy.
- tokio-tls changed a lot, they removed all the convenience methods around connecting. This resulted in updates to:
- How we listen in Mononoke Server & LFS
- How we connect in hgcli.
- Note: all this stuff has test coverage.
- The child process API changed a little bit. We used to have a ChildWrapper around the hg sync job to make a Tokio 0.2.x child look more like a Tokio 1.x Child, so now we can just remove this.
- Hyper changed their Websocket upgrade mechanism (you now need the whole `Request` to upgrade, whereas before that you needed just the `Body`, so I changed up our code a little bit in Mononoke's HTTP acceptor to defer splitting up the `Request` into parts until after we know whether we plan to upgrade it.
- I removed the MySQL tests that didn't use mysql client, because we're leaving that behind and don't intend to support it on Tokio 1.x.
Reviewed By: mitrandir77
Differential Revision: D26669620
fbshipit-source-id: acb6aff92e7f70a7a43f32cf758f252f330e60c9
2021-04-28 17:35:21 +03:00
|
|
|
"backfill_derived_data",
|
|
|
|
"benchmark_filestore",
|
2021-03-11 15:25:00 +03:00
|
|
|
"benchmarks/derived_data",
|
2021-03-11 15:25:00 +03:00
|
|
|
"benchmarks/simulated_repo",
|
mononoke: update to tokio 1.x
Summary:
NOTE: there is one final pre-requisite here, which is that we should default all Mononoke binaries to `--use-mysql-client` because the other SQL client implementations will break once this lands. That said, this is probably the right time to start reviewing.
There's a lot going on here, but Tokio updates being what they are, it has to happen as just one diff (though I did try to minimize churn by modernizing a bunch of stuff in earlier diffs).
Here's a detailed list of what is going on:
- I had to add a number `cargo_toml_dir` for binaries in `eden/mononoke/TARGETS`, because we have to use 2 versions of Bytes concurrently at this time, and the two cannot co-exist in the same Cargo workspace.
- Lots of little Tokio changes:
- Stream abstractions moving to `tokio-stream`
- `tokio::time::delay_for` became `tokio::time::sleep`
- `tokio::sync::watch::Sender::send` became `tokio::sync::watch::Sender::broadcast`
- `tokio::sync::Semaphore::acquire` returns a `Result` now.
- `tokio::runtime::Runtime::block_on` no longer takes a `&mut self` (just a `&self`).
- `Notify` grew a few more methods with different semantics. We only use this in tests, I used what seemed logical given the use case.
- Runtime builders have changed quite a bit:
- My `no_coop` patch is gone in Tokio 1.x, but it has a new `tokio::task::unconstrained` wrapper (also from me), which I included on `MononokeApi::new`.
- Tokio now detects your logical CPUs, not physical CPUs, so we no longer need to use `num_cpus::get()` to figure it out.
- Tokio 1.x now uses Bytes 1.x:
- At the edges (i.e. streams returned to Hyper or emitted by RepoClient), we need to return Bytes 1.x. However, internally we still use Bytes 0.5 in some places (notably: Filestore).
- In LFS, this means we make a copy. We used to do that a while ago anyway (in the other direction) and it was never a meaningful CPU cost, so I think this is fine.
- In Mononoke Server it doesn't really matter because that still generates ... Bytes 0.1 anyway so there was a copy before from 0.1 to 0.5 and it's from 0.1 to 1.x.
- In the very few places where we read stuff using Tokio from the outside world (historical import tools for LFS), we copy.
- tokio-tls changed a lot, they removed all the convenience methods around connecting. This resulted in updates to:
- How we listen in Mononoke Server & LFS
- How we connect in hgcli.
- Note: all this stuff has test coverage.
- The child process API changed a little bit. We used to have a ChildWrapper around the hg sync job to make a Tokio 0.2.x child look more like a Tokio 1.x Child, so now we can just remove this.
- Hyper changed their Websocket upgrade mechanism (you now need the whole `Request` to upgrade, whereas before that you needed just the `Body`, so I changed up our code a little bit in Mononoke's HTTP acceptor to defer splitting up the `Request` into parts until after we know whether we plan to upgrade it.
- I removed the MySQL tests that didn't use mysql client, because we're leaving that behind and don't intend to support it on Tokio 1.x.
Reviewed By: mitrandir77
Differential Revision: D26669620
fbshipit-source-id: acb6aff92e7f70a7a43f32cf758f252f330e60c9
2021-04-28 17:35:21 +03:00
|
|
|
"blobimport",
|
2021-02-26 02:09:13 +03:00
|
|
|
"blobimport_lib",
|
|
|
|
"blobimport_lib/consts",
|
|
|
|
"blobrepo",
|
|
|
|
"blobrepo/blobrepo_hg",
|
|
|
|
"blobrepo/blobsync",
|
|
|
|
"blobrepo/changeset_fetcher",
|
|
|
|
"blobrepo/common",
|
|
|
|
"blobrepo/errors",
|
|
|
|
"blobrepo/override",
|
|
|
|
"blobrepo/repo_blobstore",
|
|
|
|
"blobrepo_utils",
|
|
|
|
"blobstore",
|
|
|
|
"blobstore/blobstore_stats",
|
|
|
|
"blobstore/cacheblob",
|
|
|
|
"blobstore/chaosblob",
|
|
|
|
"blobstore/delayblob",
|
|
|
|
"blobstore/factory",
|
|
|
|
"blobstore/fileblob",
|
|
|
|
"blobstore/if",
|
|
|
|
"blobstore/logblob",
|
|
|
|
"blobstore/memblob",
|
|
|
|
"blobstore/multiplexedblob",
|
|
|
|
"blobstore/packblob",
|
|
|
|
"blobstore/packblob/if",
|
|
|
|
"blobstore/prefixblob",
|
|
|
|
"blobstore/readonlyblob",
|
|
|
|
"blobstore/redactedblobstore",
|
|
|
|
"blobstore/samplingblob",
|
|
|
|
"blobstore/sqlblob",
|
|
|
|
"blobstore/throttledblob",
|
|
|
|
"blobstore/virtually_sharded_blobstore",
|
mononoke: update to tokio 1.x
Summary:
NOTE: there is one final pre-requisite here, which is that we should default all Mononoke binaries to `--use-mysql-client` because the other SQL client implementations will break once this lands. That said, this is probably the right time to start reviewing.
There's a lot going on here, but Tokio updates being what they are, it has to happen as just one diff (though I did try to minimize churn by modernizing a bunch of stuff in earlier diffs).
Here's a detailed list of what is going on:
- I had to add a number `cargo_toml_dir` for binaries in `eden/mononoke/TARGETS`, because we have to use 2 versions of Bytes concurrently at this time, and the two cannot co-exist in the same Cargo workspace.
- Lots of little Tokio changes:
- Stream abstractions moving to `tokio-stream`
- `tokio::time::delay_for` became `tokio::time::sleep`
- `tokio::sync::watch::Sender::send` became `tokio::sync::watch::Sender::broadcast`
- `tokio::sync::Semaphore::acquire` returns a `Result` now.
- `tokio::runtime::Runtime::block_on` no longer takes a `&mut self` (just a `&self`).
- `Notify` grew a few more methods with different semantics. We only use this in tests, I used what seemed logical given the use case.
- Runtime builders have changed quite a bit:
- My `no_coop` patch is gone in Tokio 1.x, but it has a new `tokio::task::unconstrained` wrapper (also from me), which I included on `MononokeApi::new`.
- Tokio now detects your logical CPUs, not physical CPUs, so we no longer need to use `num_cpus::get()` to figure it out.
- Tokio 1.x now uses Bytes 1.x:
- At the edges (i.e. streams returned to Hyper or emitted by RepoClient), we need to return Bytes 1.x. However, internally we still use Bytes 0.5 in some places (notably: Filestore).
- In LFS, this means we make a copy. We used to do that a while ago anyway (in the other direction) and it was never a meaningful CPU cost, so I think this is fine.
- In Mononoke Server it doesn't really matter because that still generates ... Bytes 0.1 anyway so there was a copy before from 0.1 to 0.5 and it's from 0.1 to 1.x.
- In the very few places where we read stuff using Tokio from the outside world (historical import tools for LFS), we copy.
- tokio-tls changed a lot, they removed all the convenience methods around connecting. This resulted in updates to:
- How we listen in Mononoke Server & LFS
- How we connect in hgcli.
- Note: all this stuff has test coverage.
- The child process API changed a little bit. We used to have a ChildWrapper around the hg sync job to make a Tokio 0.2.x child look more like a Tokio 1.x Child, so now we can just remove this.
- Hyper changed their Websocket upgrade mechanism (you now need the whole `Request` to upgrade, whereas before that you needed just the `Body`, so I changed up our code a little bit in Mononoke's HTTP acceptor to defer splitting up the `Request` into parts until after we know whether we plan to upgrade it.
- I removed the MySQL tests that didn't use mysql client, because we're leaving that behind and don't intend to support it on Tokio 1.x.
Reviewed By: mitrandir77
Differential Revision: D26669620
fbshipit-source-id: acb6aff92e7f70a7a43f32cf758f252f330e60c9
2021-04-28 17:35:21 +03:00
|
|
|
"blobstore_healer",
|
2021-02-26 02:09:13 +03:00
|
|
|
"blobstore_sync_queue",
|
|
|
|
"bonsai_git_mapping",
|
|
|
|
"bonsai_git_mapping/git_mapping_pushrebase_hook",
|
|
|
|
"bonsai_globalrev_mapping",
|
|
|
|
"bonsai_globalrev_mapping/globalrev_pushrebase_hook",
|
|
|
|
"bonsai_globalrev_mapping/if",
|
|
|
|
"bonsai_hg_mapping",
|
|
|
|
"bonsai_hg_mapping/if",
|
|
|
|
"bonsai_svnrev_mapping",
|
|
|
|
"bonsai_svnrev_mapping/if",
|
|
|
|
"bookmarks",
|
|
|
|
"bookmarks/bookmarks_movement",
|
|
|
|
"bookmarks/bookmarks_types",
|
|
|
|
"bookmarks/dbbookmarks",
|
|
|
|
"bookmarks/warm_bookmarks_cache",
|
|
|
|
"bulkops",
|
|
|
|
"bulkops/bench",
|
|
|
|
"cache_warmup",
|
|
|
|
"changesets",
|
2021-04-29 16:10:17 +03:00
|
|
|
"changesets/changesets_impl",
|
2021-02-26 02:09:13 +03:00
|
|
|
"changesets/if",
|
|
|
|
"cmdlib",
|
mononoke: use MononokeEnvironment in RepoFactory
Summary:
There is a very frustrating operation that happens often when working on the
Mononoke code base:
- You want to add a flag
- You want to consume it in the repo somewhere
Unfortunately, when we need to do this, we end up having to thread this from a
million places and parse it out in every single main() we have.
This is a mess, and it results in every single Mononoke binary starting with
heaps of useless boilerplate:
```
let matches = app.get_matches();
let (caching, logger, mut runtime) = matches.init_mononoke(fb)?;
let config_store = args::init_config_store(fb, &logger, &matches)?;
let mysql_options = args::parse_mysql_options(&matches);
let blobstore_options = args::parse_blobstore_options(&matches)?;
let readonly_storage = args::parse_readonly_storage(&matches);
```
So, this diff updates us to just use MononokeEnvironment directly in
RepoFactory, which means none of that has to happen: we can now add a flag,
parse it into MononokeEnvironment, and get going.
While we're at it, we can also remove blobstore options and all that jazz from
MononokeApiEnvironment since now it's there in the underlying RepoFactory.
Reviewed By: HarveyHunt
Differential Revision: D27767700
fbshipit-source-id: e1e359bf403b4d3d7b36e5f670aa1a7dd4f1d209
2021-04-16 20:26:03 +03:00
|
|
|
"cmdlib/environment",
|
2021-02-26 02:09:13 +03:00
|
|
|
"cmdlib/x_repo",
|
2021-04-27 18:53:40 +03:00
|
|
|
"cmds/copy_blobstore_keys",
|
2021-02-26 02:09:13 +03:00
|
|
|
"commit_rewriting/backsyncer",
|
|
|
|
"commit_rewriting/bookmark_renaming",
|
|
|
|
"commit_rewriting/bookmarks_validator",
|
|
|
|
"commit_rewriting/commit_validator",
|
|
|
|
"commit_rewriting/cross_repo_sync",
|
|
|
|
"commit_rewriting/cross_repo_sync/test_utils",
|
|
|
|
"commit_rewriting/live_commit_sync_config",
|
|
|
|
"commit_rewriting/megarepo",
|
|
|
|
"commit_rewriting/mononoke_x_repo_sync_job",
|
|
|
|
"commit_rewriting/movers",
|
|
|
|
"commit_rewriting/synced_commit_mapping",
|
|
|
|
"common/allocation_tracing",
|
|
|
|
"common/async_limiter",
|
|
|
|
"common/async_limiter/examples/tokio_v2",
|
|
|
|
"common/bounded_traversal",
|
|
|
|
"common/copy_utils",
|
|
|
|
"common/dedupmap",
|
|
|
|
"common/futures_watchdog",
|
|
|
|
"common/iterhelpers",
|
mononoke/rendezvous: introduce query batching
Summary:
This introduces a basic building block for query batching. I called this
rendezvous, since it's about multiple queries meeting up in the same place :)
There are a few (somewhat conflicting) goals this tries to satisfy, so let's go
over them:
1), we'd like to reduce the total number of queries made by batch jobs. For
example, group hg bonsai lookups made by the walker. Those jobs are
characterized by the fact that they have a lot of queries to make, all the
time. Here's an example: https://fburl.com/ods/zuiep7yh.
2), we'd like to reduce the overall number of connections held to MySQL by
our tasks. The main way we achieve this is by reducing the maximum number of
concurrent queries. Indeed, a high total number of queries doesn't necessarily
result in a lot of connections as long as they're not concurrent, because we
can reuse connections. On the other hand, if you dispatch 100 concurrent
queries, that _does_ use 100 connections. This is something that applies to
batch jobs due to their query volume, but also to "interactive" jobs like
Mononoke Server or SCS, just not all the time. Here's an example:
https://fburl.com/ods/o6gp07qp (you can see the query count is overall low, but
sometimes spikes substantially).
2.1) It's also worth noting that concurrent queries are often the result of
many clients wanting the same data, so deduplication is also useful here.
3), we also don't want to impact the latency of interactive jobs when they
need to a little query here or there (i.e. it's largely fine if our jobs all
hold a few connections to MySQL and use them somewhat consistently).
4), we'd like this to make it easier to do batching right. For example, if
you have 100 Bonsais to map to hg, you should be able to just map and call
`future::try_join_all` and have that do the right thing.
5), we don't want "bad" queries to affect other queries negatively. One
example would be the occasional queries we make to Bonsai <-> Hg mapping in
`known` for thousands (if not more) of rows.
6), we want this to be easy to incorporate into the codebase.
So, how do we try to address all of this? Here's how:
- We ... do batching, and we deduplicate requests in a batch. This is the
easier bit and should address #1, #2 and #2.1, #4.
- However, batching is conditional. We notably don't batch very large requests
with the rest (addresses #5). We also don't batch small queries all the time:
we only batch if we are observing a throughput of queries that suggests we
can find some benefit in batching (this targets #3).
- Finally, we have some utilities for common cases like having to group by repo
id (this is `MultiRendezVous`), and this is all configurable via tunables
(and the default is to not do anything).
Reviewed By: StanislavGlebik
Differential Revision: D27010317
fbshipit-source-id: 4a2397255f9785c6722c02e4d419438fd0aafa07
2021-03-19 18:49:00 +03:00
|
|
|
"common/rendezvous",
|
2021-02-26 02:09:13 +03:00
|
|
|
"common/rust/caching_ext",
|
|
|
|
"common/rust/slog_ext",
|
|
|
|
"common/rust/sql_ext",
|
|
|
|
"common/scribe_ext",
|
|
|
|
"common/scuba_ext",
|
|
|
|
"common/sql_construct",
|
2021-04-12 15:21:39 +03:00
|
|
|
"common/timeseries",
|
2021-02-26 02:09:13 +03:00
|
|
|
"common/topo_sort",
|
|
|
|
"common/type_map",
|
|
|
|
"common/uniqueheap",
|
|
|
|
"derived_data",
|
|
|
|
"derived_data/blame",
|
|
|
|
"derived_data/changeset_info",
|
|
|
|
"derived_data/changeset_info/if",
|
|
|
|
"derived_data/deleted_files_manifest",
|
|
|
|
"derived_data/fastlog",
|
|
|
|
"derived_data/filenodes",
|
|
|
|
"derived_data/fsnodes",
|
|
|
|
"derived_data/mercurial_derived_data",
|
|
|
|
"derived_data/skeleton_manifest",
|
|
|
|
"derived_data/test_utils",
|
|
|
|
"derived_data/unodes",
|
|
|
|
"derived_data/utils",
|
|
|
|
"edenapi_server",
|
|
|
|
"edenapi_service",
|
|
|
|
"fastreplay",
|
|
|
|
"filenodes",
|
|
|
|
"filenodes/if",
|
|
|
|
"filestore",
|
|
|
|
"git/git-pool",
|
|
|
|
"git/git_types",
|
|
|
|
"git/git_types/if",
|
|
|
|
"git/gitimport",
|
|
|
|
"git/import_tools",
|
|
|
|
"gotham_ext",
|
|
|
|
"hgcli",
|
|
|
|
"hgproto",
|
|
|
|
"hook_tailer",
|
|
|
|
"hooks",
|
|
|
|
"hooks/content-stores",
|
|
|
|
"hooks/hook_manager_factory",
|
mononoke: update to tokio 1.x
Summary:
NOTE: there is one final pre-requisite here, which is that we should default all Mononoke binaries to `--use-mysql-client` because the other SQL client implementations will break once this lands. That said, this is probably the right time to start reviewing.
There's a lot going on here, but Tokio updates being what they are, it has to happen as just one diff (though I did try to minimize churn by modernizing a bunch of stuff in earlier diffs).
Here's a detailed list of what is going on:
- I had to add a number `cargo_toml_dir` for binaries in `eden/mononoke/TARGETS`, because we have to use 2 versions of Bytes concurrently at this time, and the two cannot co-exist in the same Cargo workspace.
- Lots of little Tokio changes:
- Stream abstractions moving to `tokio-stream`
- `tokio::time::delay_for` became `tokio::time::sleep`
- `tokio::sync::watch::Sender::send` became `tokio::sync::watch::Sender::broadcast`
- `tokio::sync::Semaphore::acquire` returns a `Result` now.
- `tokio::runtime::Runtime::block_on` no longer takes a `&mut self` (just a `&self`).
- `Notify` grew a few more methods with different semantics. We only use this in tests, I used what seemed logical given the use case.
- Runtime builders have changed quite a bit:
- My `no_coop` patch is gone in Tokio 1.x, but it has a new `tokio::task::unconstrained` wrapper (also from me), which I included on `MononokeApi::new`.
- Tokio now detects your logical CPUs, not physical CPUs, so we no longer need to use `num_cpus::get()` to figure it out.
- Tokio 1.x now uses Bytes 1.x:
- At the edges (i.e. streams returned to Hyper or emitted by RepoClient), we need to return Bytes 1.x. However, internally we still use Bytes 0.5 in some places (notably: Filestore).
- In LFS, this means we make a copy. We used to do that a while ago anyway (in the other direction) and it was never a meaningful CPU cost, so I think this is fine.
- In Mononoke Server it doesn't really matter because that still generates ... Bytes 0.1 anyway so there was a copy before from 0.1 to 0.5 and it's from 0.1 to 1.x.
- In the very few places where we read stuff using Tokio from the outside world (historical import tools for LFS), we copy.
- tokio-tls changed a lot, they removed all the convenience methods around connecting. This resulted in updates to:
- How we listen in Mononoke Server & LFS
- How we connect in hgcli.
- Note: all this stuff has test coverage.
- The child process API changed a little bit. We used to have a ChildWrapper around the hg sync job to make a Tokio 0.2.x child look more like a Tokio 1.x Child, so now we can just remove this.
- Hyper changed their Websocket upgrade mechanism (you now need the whole `Request` to upgrade, whereas before that you needed just the `Body`, so I changed up our code a little bit in Mononoke's HTTP acceptor to defer splitting up the `Request` into parts until after we know whether we plan to upgrade it.
- I removed the MySQL tests that didn't use mysql client, because we're leaving that behind and don't intend to support it on Tokio 1.x.
Reviewed By: mitrandir77
Differential Revision: D26669620
fbshipit-source-id: acb6aff92e7f70a7a43f32cf758f252f330e60c9
2021-04-28 17:35:21 +03:00
|
|
|
"lfs_import",
|
2021-02-26 02:09:13 +03:00
|
|
|
"lfs_import_lib",
|
|
|
|
"lfs_protocol",
|
|
|
|
"lfs_server",
|
|
|
|
"load_limiter",
|
|
|
|
"manifest",
|
megarepo: basic version of async-requests crate
Summary:
This crate is a foundation for the async requests support in megarepo service.
The idea is to be able to store serialized parameters in the blobstore upon
request arrival, and to be able to query request results from the blobstore
while polling.
This diff manipulates the following classes of types:
- param types for async methods: self-explanatory
- response types: these contain only a resulting value of a completed successful execution
- stored result types: these contain a result value of a completed execution. It may either be successful or failed. These types exist for the purpose of preserving execution result in the blobstore.
- poll-response types: these contain and option of a response. If the optional value is empty, this means that the request is not yet ready
- polling tokens: these are used by the client to ask about the processing status for a submitted request
Apart from that, some of these types have both Rust and Thrift counterparts, mainly for the purposes of us being able to implement traits for Rust types.
Relationships between these types are encoded in various traits and their associated types.
The lifecycle of an async request is as follows therefore:
1. the request is submitted by the client, and enqueued
1. params are serialized and saved into a blobstore
1. an entry is created in the SQL table
1. the key from that table is used to create a polling token
1. some external system processes a request [completely absent form this diff]
1. it notices a new entry in the queue
1. it reads request's params from the blobstore
1. it processes the request
1. it preserves either a success of a failure of the request into the blobstore
1. it updates the SQL table to mention that the request is now ready to be polled
1. the client polls the request
1. queue struct receives a polling token
1. out of that token it constructs DB keys
1. it looks up the request row and checks if it is in the ready state
1. if that is the case, it reads the result_blobstore_key value and fetches serialized result object
1. now it has to turn this serialized result into a poll response:
1. if the result is absent, poll response is a success with an empty payload
1. if the result is present and successful, poll response is a success with the result's successful variant as a payload
1. if the result is present and is a failure, the polling call throws a thrift exception with that failure
Note: Why is there yet another .thrift file introduced in this diff? I felt like these types aren't a part of the scs interface, so they don't belong in `source_control.thrift`. On the other hand, they wrap things defined in `source_control.thrift,` so I needed to include it.
Reviewed By: StanislavGlebik
Differential Revision: D27964822
fbshipit-source-id: fc1a33a799d01c908bbe18a5394eba034b780174
2021-05-10 16:50:29 +03:00
|
|
|
"megarepo_api/async_requests",
|
2021-05-10 21:47:19 +03:00
|
|
|
"megarepo_api/commit_transformation",
|
2021-04-19 18:33:00 +03:00
|
|
|
"megarepo_api/configo_client",
|
megarepo: basic version of async-requests crate
Summary:
This crate is a foundation for the async requests support in megarepo service.
The idea is to be able to store serialized parameters in the blobstore upon
request arrival, and to be able to query request results from the blobstore
while polling.
This diff manipulates the following classes of types:
- param types for async methods: self-explanatory
- response types: these contain only a resulting value of a completed successful execution
- stored result types: these contain a result value of a completed execution. It may either be successful or failed. These types exist for the purpose of preserving execution result in the blobstore.
- poll-response types: these contain and option of a response. If the optional value is empty, this means that the request is not yet ready
- polling tokens: these are used by the client to ask about the processing status for a submitted request
Apart from that, some of these types have both Rust and Thrift counterparts, mainly for the purposes of us being able to implement traits for Rust types.
Relationships between these types are encoded in various traits and their associated types.
The lifecycle of an async request is as follows therefore:
1. the request is submitted by the client, and enqueued
1. params are serialized and saved into a blobstore
1. an entry is created in the SQL table
1. the key from that table is used to create a polling token
1. some external system processes a request [completely absent form this diff]
1. it notices a new entry in the queue
1. it reads request's params from the blobstore
1. it processes the request
1. it preserves either a success of a failure of the request into the blobstore
1. it updates the SQL table to mention that the request is now ready to be polled
1. the client polls the request
1. queue struct receives a polling token
1. out of that token it constructs DB keys
1. it looks up the request row and checks if it is in the ready state
1. if that is the case, it reads the result_blobstore_key value and fetches serialized result object
1. now it has to turn this serialized result into a poll response:
1. if the result is absent, poll response is a success with an empty payload
1. if the result is present and successful, poll response is a success with the result's successful variant as a payload
1. if the result is present and is a failure, the polling call throws a thrift exception with that failure
Note: Why is there yet another .thrift file introduced in this diff? I felt like these types aren't a part of the scs interface, so they don't belong in `source_control.thrift`. On the other hand, they wrap things defined in `source_control.thrift,` so I needed to include it.
Reviewed By: StanislavGlebik
Differential Revision: D27964822
fbshipit-source-id: fc1a33a799d01c908bbe18a5394eba034b780174
2021-05-10 16:50:29 +03:00
|
|
|
"megarepo_api/if",
|
2021-05-11 12:53:14 +03:00
|
|
|
"megarepo_api/mapping",
|
2021-04-22 12:12:40 +03:00
|
|
|
"megarepo_api/megarepo_config",
|
2021-04-21 15:00:58 +03:00
|
|
|
"megarepo_api/megarepo_error",
|
2021-05-06 21:32:31 +03:00
|
|
|
"megarepo_api/requests_table",
|
2021-02-26 02:09:13 +03:00
|
|
|
"mercurial/bundle_replay_data",
|
|
|
|
"mercurial/bundles",
|
|
|
|
"mercurial/mutation",
|
|
|
|
"mercurial/revlog",
|
|
|
|
"mercurial/types",
|
|
|
|
"mercurial/types/if",
|
|
|
|
"mercurial/types/mocks",
|
|
|
|
"metaconfig/parser",
|
|
|
|
"metaconfig/types",
|
|
|
|
"microwave",
|
|
|
|
"microwave/builder",
|
|
|
|
"microwave/if",
|
|
|
|
"mononoke_api",
|
|
|
|
"mononoke_api_hg",
|
|
|
|
"mononoke_hg_sync_job",
|
|
|
|
"mononoke_types",
|
|
|
|
"mononoke_types/if",
|
|
|
|
"mononoke_types/mocks",
|
|
|
|
"mutable_counters",
|
|
|
|
"newfilenodes",
|
|
|
|
"observability",
|
|
|
|
"permission_checker",
|
|
|
|
"phases",
|
|
|
|
"pushrebase",
|
2021-04-27 13:51:52 +03:00
|
|
|
"pushrebase/pushrebase_hook",
|
2021-04-27 13:51:52 +03:00
|
|
|
"pushrebase_mutation_mapping",
|
2021-04-16 12:25:01 +03:00
|
|
|
"quiet_stream",
|
2021-02-26 02:09:13 +03:00
|
|
|
"reachabilityindex",
|
|
|
|
"reachabilityindex/common",
|
|
|
|
"reachabilityindex/if",
|
|
|
|
"reachabilityindex/skiplist",
|
|
|
|
"reachabilityindex/test-helpers",
|
|
|
|
"regenerate_hg_filenodes",
|
2021-03-22 17:23:06 +03:00
|
|
|
"repo_attributes/repo_derived_data",
|
2021-03-22 17:23:06 +03:00
|
|
|
"repo_attributes/repo_identity",
|
2021-02-26 02:09:13 +03:00
|
|
|
"repo_client",
|
|
|
|
"repo_client/getbundle_response",
|
|
|
|
"repo_client/mononoke_repo",
|
|
|
|
"repo_client/obsolete",
|
|
|
|
"repo_client/remotefilelog",
|
|
|
|
"repo_client/repo_read_write_status",
|
|
|
|
"repo_client/reverse_filler_queue",
|
|
|
|
"repo_client/scribe_commit_queue",
|
|
|
|
"repo_client/streaming_clone",
|
|
|
|
"repo_client/unbundle",
|
|
|
|
"repo_client/wirepack",
|
2021-04-07 23:59:27 +03:00
|
|
|
"repo_factory",
|
2021-03-25 17:32:21 +03:00
|
|
|
"repo_factory/test_repo_factory",
|
2021-02-26 02:09:13 +03:00
|
|
|
"repo_import",
|
|
|
|
"revset",
|
|
|
|
"revset/revset-test-helper",
|
2021-05-10 15:52:39 +03:00
|
|
|
"scs/if",
|
2021-02-26 02:09:13 +03:00
|
|
|
"segmented_changelog",
|
|
|
|
"segmented_changelog/bench/concurrent_idmap",
|
2021-03-22 17:23:06 +03:00
|
|
|
"segmented_changelog/types",
|
2021-02-26 02:09:13 +03:00
|
|
|
"server",
|
|
|
|
"server/context",
|
|
|
|
"server/monitoring",
|
2021-03-22 20:10:53 +03:00
|
|
|
"server/qps",
|
2021-02-26 02:09:13 +03:00
|
|
|
"server/repo_listener",
|
|
|
|
"server/session_id",
|
|
|
|
"sshrelay",
|
mononoke: update to tokio 1.x
Summary:
NOTE: there is one final pre-requisite here, which is that we should default all Mononoke binaries to `--use-mysql-client` because the other SQL client implementations will break once this lands. That said, this is probably the right time to start reviewing.
There's a lot going on here, but Tokio updates being what they are, it has to happen as just one diff (though I did try to minimize churn by modernizing a bunch of stuff in earlier diffs).
Here's a detailed list of what is going on:
- I had to add a number `cargo_toml_dir` for binaries in `eden/mononoke/TARGETS`, because we have to use 2 versions of Bytes concurrently at this time, and the two cannot co-exist in the same Cargo workspace.
- Lots of little Tokio changes:
- Stream abstractions moving to `tokio-stream`
- `tokio::time::delay_for` became `tokio::time::sleep`
- `tokio::sync::watch::Sender::send` became `tokio::sync::watch::Sender::broadcast`
- `tokio::sync::Semaphore::acquire` returns a `Result` now.
- `tokio::runtime::Runtime::block_on` no longer takes a `&mut self` (just a `&self`).
- `Notify` grew a few more methods with different semantics. We only use this in tests, I used what seemed logical given the use case.
- Runtime builders have changed quite a bit:
- My `no_coop` patch is gone in Tokio 1.x, but it has a new `tokio::task::unconstrained` wrapper (also from me), which I included on `MononokeApi::new`.
- Tokio now detects your logical CPUs, not physical CPUs, so we no longer need to use `num_cpus::get()` to figure it out.
- Tokio 1.x now uses Bytes 1.x:
- At the edges (i.e. streams returned to Hyper or emitted by RepoClient), we need to return Bytes 1.x. However, internally we still use Bytes 0.5 in some places (notably: Filestore).
- In LFS, this means we make a copy. We used to do that a while ago anyway (in the other direction) and it was never a meaningful CPU cost, so I think this is fine.
- In Mononoke Server it doesn't really matter because that still generates ... Bytes 0.1 anyway so there was a copy before from 0.1 to 0.5 and it's from 0.1 to 1.x.
- In the very few places where we read stuff using Tokio from the outside world (historical import tools for LFS), we copy.
- tokio-tls changed a lot, they removed all the convenience methods around connecting. This resulted in updates to:
- How we listen in Mononoke Server & LFS
- How we connect in hgcli.
- Note: all this stuff has test coverage.
- The child process API changed a little bit. We used to have a ChildWrapper around the hg sync job to make a Tokio 0.2.x child look more like a Tokio 1.x Child, so now we can just remove this.
- Hyper changed their Websocket upgrade mechanism (you now need the whole `Request` to upgrade, whereas before that you needed just the `Body`, so I changed up our code a little bit in Mononoke's HTTP acceptor to defer splitting up the `Request` into parts until after we know whether we plan to upgrade it.
- I removed the MySQL tests that didn't use mysql client, because we're leaving that behind and don't intend to support it on Tokio 1.x.
Reviewed By: mitrandir77
Differential Revision: D26669620
fbshipit-source-id: acb6aff92e7f70a7a43f32cf758f252f330e60c9
2021-04-28 17:35:21 +03:00
|
|
|
"statistics_collector",
|
2021-03-13 01:45:03 +03:00
|
|
|
"streaming_clone",
|
2021-02-26 02:09:13 +03:00
|
|
|
"tests/fixtures",
|
|
|
|
"tests/utils",
|
|
|
|
"tests/write_stub_log_entry",
|
|
|
|
"time_window_counter",
|
|
|
|
"tunables",
|
|
|
|
"tunables/tunables-derive",
|
|
|
|
"unbundle_replay",
|
mononoke: update to tokio 1.x
Summary:
NOTE: there is one final pre-requisite here, which is that we should default all Mononoke binaries to `--use-mysql-client` because the other SQL client implementations will break once this lands. That said, this is probably the right time to start reviewing.
There's a lot going on here, but Tokio updates being what they are, it has to happen as just one diff (though I did try to minimize churn by modernizing a bunch of stuff in earlier diffs).
Here's a detailed list of what is going on:
- I had to add a number `cargo_toml_dir` for binaries in `eden/mononoke/TARGETS`, because we have to use 2 versions of Bytes concurrently at this time, and the two cannot co-exist in the same Cargo workspace.
- Lots of little Tokio changes:
- Stream abstractions moving to `tokio-stream`
- `tokio::time::delay_for` became `tokio::time::sleep`
- `tokio::sync::watch::Sender::send` became `tokio::sync::watch::Sender::broadcast`
- `tokio::sync::Semaphore::acquire` returns a `Result` now.
- `tokio::runtime::Runtime::block_on` no longer takes a `&mut self` (just a `&self`).
- `Notify` grew a few more methods with different semantics. We only use this in tests, I used what seemed logical given the use case.
- Runtime builders have changed quite a bit:
- My `no_coop` patch is gone in Tokio 1.x, but it has a new `tokio::task::unconstrained` wrapper (also from me), which I included on `MononokeApi::new`.
- Tokio now detects your logical CPUs, not physical CPUs, so we no longer need to use `num_cpus::get()` to figure it out.
- Tokio 1.x now uses Bytes 1.x:
- At the edges (i.e. streams returned to Hyper or emitted by RepoClient), we need to return Bytes 1.x. However, internally we still use Bytes 0.5 in some places (notably: Filestore).
- In LFS, this means we make a copy. We used to do that a while ago anyway (in the other direction) and it was never a meaningful CPU cost, so I think this is fine.
- In Mononoke Server it doesn't really matter because that still generates ... Bytes 0.1 anyway so there was a copy before from 0.1 to 0.5 and it's from 0.1 to 1.x.
- In the very few places where we read stuff using Tokio from the outside world (historical import tools for LFS), we copy.
- tokio-tls changed a lot, they removed all the convenience methods around connecting. This resulted in updates to:
- How we listen in Mononoke Server & LFS
- How we connect in hgcli.
- Note: all this stuff has test coverage.
- The child process API changed a little bit. We used to have a ChildWrapper around the hg sync job to make a Tokio 0.2.x child look more like a Tokio 1.x Child, so now we can just remove this.
- Hyper changed their Websocket upgrade mechanism (you now need the whole `Request` to upgrade, whereas before that you needed just the `Body`, so I changed up our code a little bit in Mononoke's HTTP acceptor to defer splitting up the `Request` into parts until after we know whether we plan to upgrade it.
- I removed the MySQL tests that didn't use mysql client, because we're leaving that behind and don't intend to support it on Tokio 1.x.
Reviewed By: mitrandir77
Differential Revision: D26669620
fbshipit-source-id: acb6aff92e7f70a7a43f32cf758f252f330e60c9
2021-04-28 17:35:21 +03:00
|
|
|
"upload_globalrevs",
|
2021-02-26 02:09:13 +03:00
|
|
|
"walker",
|
|
|
|
]
|