2020-06-17 16:26:05 +03:00
|
|
|
[package]
|
|
|
|
name = "eden_mononoke"
|
|
|
|
version = "0.1.0"
|
2021-02-19 22:02:06 +03:00
|
|
|
authors = ["Facebook"]
|
|
|
|
edition = "2018"
|
2020-06-17 16:26:05 +03:00
|
|
|
license = "GPLv2+"
|
2020-07-11 08:08:09 +03:00
|
|
|
|
|
|
|
[[bin]]
|
|
|
|
name = "admin"
|
|
|
|
path = "cmds/admin/main.rs"
|
2020-06-17 16:26:05 +03:00
|
|
|
|
|
|
|
[[bin]]
|
|
|
|
name = "aliasverify"
|
|
|
|
path = "cmds/aliasverify.rs"
|
|
|
|
|
|
|
|
[[bin]]
|
|
|
|
name = "backfill_derived_data"
|
|
|
|
path = "cmds/backfill_derived_data/main.rs"
|
|
|
|
|
|
|
|
[[bin]]
|
2021-02-17 23:40:17 +03:00
|
|
|
name = "backfill_mapping"
|
|
|
|
path = "cmds/backfill_mapping.rs"
|
2020-06-17 16:26:05 +03:00
|
|
|
|
2020-07-11 08:08:09 +03:00
|
|
|
[[bin]]
|
|
|
|
name = "benchmark_filestore"
|
|
|
|
path = "cmds/benchmark_filestore.rs"
|
2021-03-02 17:41:22 +03:00
|
|
|
test = false
|
2020-07-11 08:08:09 +03:00
|
|
|
|
2020-06-17 16:26:05 +03:00
|
|
|
[[bin]]
|
|
|
|
name = "benchmark_storage_config"
|
|
|
|
path = "cmds/benchmark_storage_config/main.rs"
|
|
|
|
|
2020-07-13 12:58:20 +03:00
|
|
|
[[bin]]
|
|
|
|
name = "blobimport"
|
|
|
|
path = "cmds/blobimport.rs"
|
|
|
|
|
2020-07-13 12:58:20 +03:00
|
|
|
[[bin]]
|
|
|
|
name = "blobstore_healer"
|
|
|
|
path = "cmds/blobstore_healer/main.rs"
|
|
|
|
|
2020-06-17 16:26:05 +03:00
|
|
|
[[bin]]
|
|
|
|
name = "bonsai_verify"
|
|
|
|
path = "cmds/bonsai_verify/main.rs"
|
2021-03-02 17:41:22 +03:00
|
|
|
test = false
|
2020-06-17 16:26:05 +03:00
|
|
|
|
|
|
|
[[bin]]
|
|
|
|
name = "configlint"
|
|
|
|
path = "cmds/configlint.rs"
|
|
|
|
|
|
|
|
[[bin]]
|
|
|
|
name = "dumprev"
|
|
|
|
path = "cmds/dumprev.rs"
|
2021-03-02 17:41:22 +03:00
|
|
|
test = false
|
2020-06-17 16:26:05 +03:00
|
|
|
|
|
|
|
[[bin]]
|
|
|
|
name = "idxdump"
|
|
|
|
path = "cmds/idxdump.rs"
|
2021-03-02 17:41:22 +03:00
|
|
|
test = false
|
2020-06-17 16:26:05 +03:00
|
|
|
|
|
|
|
[[bin]]
|
|
|
|
name = "lfs_import"
|
|
|
|
path = "cmds/lfs_import.rs"
|
2021-03-02 17:41:22 +03:00
|
|
|
test = false
|
2020-06-17 16:26:05 +03:00
|
|
|
|
2020-07-10 10:59:29 +03:00
|
|
|
[[bin]]
|
|
|
|
name = "manual_scrub"
|
|
|
|
path = "cmds/manual_scrub/main.rs"
|
|
|
|
|
2021-04-20 22:44:12 +03:00
|
|
|
[[bin]]
|
|
|
|
name = "packer"
|
|
|
|
path = "cmds/packer/main.rs"
|
|
|
|
|
2020-06-17 16:26:05 +03:00
|
|
|
[[bin]]
|
|
|
|
name = "rechunker"
|
|
|
|
path = "cmds/rechunker.rs"
|
|
|
|
|
|
|
|
[[bin]]
|
|
|
|
name = "revlogrepo"
|
|
|
|
path = "cmds/revlogrepo.rs"
|
|
|
|
|
2020-10-08 19:41:16 +03:00
|
|
|
[[bin]]
|
|
|
|
name = "segmented_changelog_seeder"
|
|
|
|
path = "cmds/segmented_changelog_seeder.rs"
|
|
|
|
|
2020-10-28 20:22:18 +03:00
|
|
|
[[bin]]
|
2020-10-30 03:35:41 +03:00
|
|
|
name = "segmented_changelog_tailer"
|
|
|
|
path = "cmds/segmented_changelog_tailer.rs"
|
|
|
|
|
|
|
|
[[bin]]
|
2020-10-28 20:22:18 +03:00
|
|
|
name = "sqlblob_gc"
|
|
|
|
path = "cmds/sqlblob_gc/main.rs"
|
|
|
|
|
2020-06-17 16:26:05 +03:00
|
|
|
[[bin]]
|
|
|
|
name = "statistics_collector"
|
|
|
|
path = "cmds/statistics_collector.rs"
|
|
|
|
|
2020-09-01 17:12:10 +03:00
|
|
|
[[bin]]
|
|
|
|
name = "streaming_clone_warmup"
|
|
|
|
path = "cmds/streaming_clone_warmup/main.rs"
|
|
|
|
|
2020-06-17 16:26:05 +03:00
|
|
|
[[bin]]
|
|
|
|
name = "upload_globalrevs"
|
|
|
|
path = "cmds/upload_globalrevs.rs"
|
|
|
|
|
|
|
|
[dependencies]
|
2021-02-19 22:02:06 +03:00
|
|
|
anyhow = "1.0"
|
|
|
|
ascii = "1.0"
|
2021-04-01 17:01:00 +03:00
|
|
|
async-compression = { version = "0.3", features = ["all"] }
|
2021-03-06 08:45:20 +03:00
|
|
|
async-trait = "0.1.45"
|
2021-02-26 02:09:13 +03:00
|
|
|
backsyncer = { version = "0.1.0", path = "commit_rewriting/backsyncer" }
|
|
|
|
blame = { version = "0.1.0", path = "derived_data/blame" }
|
|
|
|
blobimport_lib = { version = "0.1.0", path = "blobimport_lib" }
|
|
|
|
blobrepo = { version = "0.1.0", path = "blobrepo" }
|
|
|
|
blobrepo_hg = { version = "0.1.0", path = "blobrepo/blobrepo_hg" }
|
|
|
|
blobrepo_override = { version = "0.1.0", path = "blobrepo/override" }
|
|
|
|
blobrepo_utils = { version = "0.1.0", path = "blobrepo_utils" }
|
|
|
|
blobstore = { version = "0.1.0", path = "blobstore" }
|
|
|
|
blobstore_factory = { version = "0.1.0", path = "blobstore/factory" }
|
|
|
|
blobstore_sync_queue = { version = "0.1.0", path = "blobstore_sync_queue" }
|
|
|
|
bonsai_globalrev_mapping = { version = "0.1.0", path = "bonsai_globalrev_mapping" }
|
|
|
|
bookmark_renaming = { version = "0.1.0", path = "commit_rewriting/bookmark_renaming" }
|
|
|
|
bookmarks = { version = "0.1.0", path = "bookmarks" }
|
|
|
|
bookmarks_movement = { version = "0.1.0", path = "bookmarks/bookmarks_movement" }
|
|
|
|
borrowed = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
bounded_traversal = { version = "0.1.0", path = "common/bounded_traversal" }
|
|
|
|
bulkops = { version = "0.1.0", path = "bulkops" }
|
2021-02-19 22:02:06 +03:00
|
|
|
bytes = { version = "0.5", features = ["serde"] }
|
|
|
|
bytesize = "1.0"
|
2021-02-26 02:09:13 +03:00
|
|
|
cacheblob = { version = "0.1.0", path = "blobstore/cacheblob" }
|
|
|
|
cached_config = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
cachelib = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
changeset_fetcher = { version = "0.1.0", path = "blobrepo/changeset_fetcher" }
|
|
|
|
changesets = { version = "0.1.0", path = "changesets" }
|
2021-02-19 22:02:06 +03:00
|
|
|
chrono = { version = "0.4", features = ["serde"] }
|
|
|
|
clap = "2.33"
|
2021-02-26 02:09:13 +03:00
|
|
|
cloned = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
cmdlib = { version = "0.1.0", path = "cmdlib" }
|
|
|
|
context = { version = "0.1.0", path = "server/context" }
|
|
|
|
copy_utils = { version = "0.1.0", path = "common/copy_utils" }
|
2021-02-19 22:02:06 +03:00
|
|
|
criterion = "=0.3.1"
|
2021-02-26 02:09:13 +03:00
|
|
|
cross_repo_sync = { version = "0.1.0", path = "commit_rewriting/cross_repo_sync" }
|
|
|
|
dbbookmarks = { version = "0.1.0", path = "bookmarks/dbbookmarks" }
|
|
|
|
deleted_files_manifest = { version = "0.1.0", path = "derived_data/deleted_files_manifest" }
|
2021-04-06 18:42:33 +03:00
|
|
|
derive_more = "0.99.3"
|
2021-02-26 02:09:13 +03:00
|
|
|
derived_data = { version = "0.1.0", path = "derived_data" }
|
|
|
|
derived_data_filenodes = { version = "0.1.0", path = "derived_data/filenodes" }
|
|
|
|
derived_data_utils = { version = "0.1.0", path = "derived_data/utils" }
|
mononoke: use MononokeEnvironment in RepoFactory
Summary:
There is a very frustrating operation that happens often when working on the
Mononoke code base:
- You want to add a flag
- You want to consume it in the repo somewhere
Unfortunately, when we need to do this, we end up having to thread this from a
million places and parse it out in every single main() we have.
This is a mess, and it results in every single Mononoke binary starting with
heaps of useless boilerplate:
```
let matches = app.get_matches();
let (caching, logger, mut runtime) = matches.init_mononoke(fb)?;
let config_store = args::init_config_store(fb, &logger, &matches)?;
let mysql_options = args::parse_mysql_options(&matches);
let blobstore_options = args::parse_blobstore_options(&matches)?;
let readonly_storage = args::parse_readonly_storage(&matches);
```
So, this diff updates us to just use MononokeEnvironment directly in
RepoFactory, which means none of that has to happen: we can now add a flag,
parse it into MononokeEnvironment, and get going.
While we're at it, we can also remove blobstore options and all that jazz from
MononokeApiEnvironment since now it's there in the underlying RepoFactory.
Reviewed By: HarveyHunt
Differential Revision: D27767700
fbshipit-source-id: e1e359bf403b4d3d7b36e5f670aa1a7dd4f1d209
2021-04-16 20:26:03 +03:00
|
|
|
environment = { version = "0.1.0", path = "cmdlib/environment" }
|
2021-02-26 02:09:13 +03:00
|
|
|
failure_ext = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
fastlog = { version = "0.1.0", path = "derived_data/fastlog" }
|
|
|
|
fbinit = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
fbthrift = { version = "0.0.1+unstable", git = "https://github.com/facebook/fbthrift.git", branch = "master" }
|
|
|
|
filenodes = { version = "0.1.0", path = "filenodes" }
|
|
|
|
filestore = { version = "0.1.0", path = "filestore" }
|
|
|
|
fsnodes = { version = "0.1.0", path = "derived_data/fsnodes" }
|
2021-03-04 17:40:30 +03:00
|
|
|
futures = { version = "0.3.13", features = ["async-await", "compat"] }
|
|
|
|
futures-old = { package = "futures", version = "0.1.31" }
|
2021-02-19 22:02:06 +03:00
|
|
|
futures-util = "0.3.7"
|
2021-02-26 02:09:13 +03:00
|
|
|
futures_03_ext = { package = "futures_ext", version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
futures_ext = { package = "futures_01_ext", version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
futures_stats = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
git_types = { version = "0.1.0", path = "git/git_types" }
|
2021-02-19 22:02:06 +03:00
|
|
|
humantime = "1.3"
|
|
|
|
itertools = "0.8"
|
|
|
|
lazy_static = "1.0"
|
2021-02-26 02:09:13 +03:00
|
|
|
lfs_import_lib = { version = "0.1.0", path = "lfs_import_lib" }
|
|
|
|
live_commit_sync_config = { version = "0.1.0", path = "commit_rewriting/live_commit_sync_config" }
|
|
|
|
lock_ext = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
manifest = { version = "0.1.0", path = "manifest" }
|
2021-02-19 22:02:06 +03:00
|
|
|
maplit = "1.0"
|
2021-02-26 02:09:13 +03:00
|
|
|
memblob = { version = "0.1.0", path = "blobstore/memblob" }
|
|
|
|
mercurial_bundle_replay_data = { version = "0.1.0", path = "mercurial/bundle_replay_data" }
|
|
|
|
mercurial_derived_data = { version = "0.1.0", path = "derived_data/mercurial_derived_data" }
|
|
|
|
mercurial_revlog = { version = "0.1.0", path = "mercurial/revlog" }
|
|
|
|
mercurial_types = { version = "0.1.0", path = "mercurial/types" }
|
|
|
|
metaconfig_types = { version = "0.1.0", path = "metaconfig/types" }
|
|
|
|
mononoke_hg_sync_job_helper_lib = { version = "0.1.0", path = "mononoke_hg_sync_job" }
|
|
|
|
mononoke_types = { version = "0.1.0", path = "mononoke_types" }
|
|
|
|
mutable_counters = { version = "0.1.0", path = "mutable_counters" }
|
2021-04-20 22:44:12 +03:00
|
|
|
packblob = { version = "0.1.0", path = "blobstore/packblob" }
|
2021-02-26 02:09:13 +03:00
|
|
|
prefixblob = { version = "0.1.0", path = "blobstore/prefixblob" }
|
|
|
|
pushrebase = { version = "0.1.0", path = "pushrebase" }
|
2021-02-19 22:02:06 +03:00
|
|
|
rand = { version = "0.7", features = ["small_rng"] }
|
2021-02-26 02:09:13 +03:00
|
|
|
redactedblobstore = { version = "0.1.0", path = "blobstore/redactedblobstore" }
|
2021-02-19 22:02:06 +03:00
|
|
|
regex = "1.4.2"
|
2021-04-07 23:59:27 +03:00
|
|
|
repo_factory = { version = "0.1.0", path = "repo_factory" }
|
2021-02-26 02:09:13 +03:00
|
|
|
revset = { version = "0.1.0", path = "revset" }
|
|
|
|
scuba_ext = { version = "0.1.0", path = "common/scuba_ext" }
|
|
|
|
segmented_changelog = { version = "0.1.0", path = "segmented_changelog" }
|
2021-02-23 05:14:45 +03:00
|
|
|
serde = { version = "=1.0.118", features = ["derive", "rc"] }
|
2021-02-19 22:02:06 +03:00
|
|
|
serde_derive = "1.0"
|
|
|
|
serde_json = { version = "1.0", features = ["float_roundtrip"] }
|
|
|
|
sha2 = "0.8"
|
2021-02-26 02:09:13 +03:00
|
|
|
skeleton_manifest = { version = "0.1.0", path = "derived_data/skeleton_manifest" }
|
|
|
|
skiplist = { version = "0.1.0", path = "reachabilityindex/skiplist" }
|
2021-02-19 22:02:06 +03:00
|
|
|
slog = { version = "2.5", features = ["max_level_debug"] }
|
2021-03-11 15:25:00 +03:00
|
|
|
sorted_vector_map = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
2021-02-26 02:09:13 +03:00
|
|
|
sql_construct = { version = "0.1.0", path = "common/sql_construct" }
|
|
|
|
sql_ext = { version = "0.1.0", path = "common/rust/sql_ext" }
|
|
|
|
sqlblob = { version = "0.1.0", path = "blobstore/sqlblob" }
|
|
|
|
stats = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
|
|
|
streaming_clone = { version = "0.1.0", path = "repo_client/streaming_clone" }
|
|
|
|
synced_commit_mapping = { version = "0.1.0", path = "commit_rewriting/synced_commit_mapping" }
|
2021-04-07 19:29:57 +03:00
|
|
|
tempfile = "3.1"
|
2021-02-19 22:02:06 +03:00
|
|
|
thiserror = "1.0"
|
2021-02-26 02:09:13 +03:00
|
|
|
throttledblob = { version = "0.1.0", path = "blobstore/throttledblob" }
|
|
|
|
time_ext = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
2021-02-12 15:54:03 +03:00
|
|
|
tokio = { version = "0.2.25", features = ["full", "test-util"] }
|
2021-02-25 18:40:23 +03:00
|
|
|
tokio-util = { version = "0.3", features = ["codec", "udp"] }
|
2021-02-26 02:09:13 +03:00
|
|
|
tokio_shim = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
2020-12-17 18:43:34 +03:00
|
|
|
toml = "=0.5.7"
|
2021-02-26 02:09:13 +03:00
|
|
|
unodes = { version = "0.1.0", path = "derived_data/unodes" }
|
|
|
|
xdiff = { version = "0.1.0", path = "../scm/lib/xdiff" }
|
2020-06-17 16:26:05 +03:00
|
|
|
|
|
|
|
[dev-dependencies]
|
common/rust: use fbinit-tokio
Summary:
This diffs add a layer of indirection between fbinit and tokio, thus allowing
us to use fbinit with tokio 0.2 or tokio 1.x.
The way this works is that you specify the Tokio you want by adding it as an
extra dependency alongside `fbinit` in your `TARGETS` (before this, you had to
always include `tokio-02`).
If you use `fbinit-tokio`, then `#[fbinit::main]` and `#[fbinit::test]` get you
a Tokio 1.x runtime, whereas if you use `fbinit-tokio-02`, you get a Tokio 0.2
runtime.
This diff is big, because it needs to change all the TARGETS that reference
this in the same diff that introduces the mechanism. I also didn't produce it
by hand.
Instead, I scripted the transformation using this script: P242773846
I then ran it using:
```
{ hg grep -l "fbinit::test"; hg grep -l "fbinit::main" } | \
sort | \
uniq | \
xargs ~/codemod/codemod.py \
&& yes | arc lint \
&& common/rust/cargo_from_buck/bin/autocargo
```
Finally, I grabbed the files returned by `hg grep`, then fed them to:
```
arc lint-rust --paths-from ~/files2 --apply-patches --take RUSTFIXDEPS
```
(I had to modify the file list a bit: notably I removed stuff from scripts/ because
some of that causes Buck to crash when running lint-rust, and I also had to add
fbcode/ as a prefix everywhere).
Reviewed By: mitrandir77
Differential Revision: D26754757
fbshipit-source-id: 326b1c4efc9a57ea89db9b1d390677bcd2ab985e
2021-03-03 15:06:49 +03:00
|
|
|
fbinit-tokio-02 = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
2021-02-26 02:09:13 +03:00
|
|
|
fixtures = { version = "0.1.0", path = "tests/fixtures" }
|
2021-03-25 17:32:21 +03:00
|
|
|
test_repo_factory = { version = "0.1.0", path = "repo_factory/test_repo_factory" }
|
2021-02-26 02:09:13 +03:00
|
|
|
tests_utils = { version = "0.1.0", path = "tests/utils" }
|
2020-06-17 16:26:05 +03:00
|
|
|
|
|
|
|
[patch.crates-io]
|
2021-02-26 02:09:13 +03:00
|
|
|
addr2line = { git = "https://github.com/gimli-rs/addr2line.git", rev = "0b6b6018b5b252a18e628fba03885f7d21844b3c" }
|
2021-04-23 00:32:44 +03:00
|
|
|
async-compression = { git = "https://github.com/ahornby/async-compression", rev = "c7fe7f9512e742f0c0097a19de0fca89da140b31" }
|
2020-11-19 23:56:30 +03:00
|
|
|
bytecount = { git = "https://github.com/llogiq/bytecount", rev = "469eaf8395c99397cd64d059737a9054aa014088" }
|
2020-06-17 16:26:05 +03:00
|
|
|
chashmap = { git = "https://gitlab.redox-os.org/ahornby/chashmap", rev = "901ace2ca3cdbc2095adb1af111d211e254e2aae" }
|
|
|
|
const-random = { git = "https://github.com/fbsource/const-random", rev = "374c5b46427fe2ffbf6acbd9c1687e0f1a809f95" }
|
2021-04-08 21:48:52 +03:00
|
|
|
curl = { git = "https://github.com/kulshrax/curl-rust", rev = "2a15bbd8dbbd54734313fa703a64db7ce6ddaff0" }
|
|
|
|
curl-sys = { git = "https://github.com/kulshrax/curl-rust", rev = "2a15bbd8dbbd54734313fa703a64db7ce6ddaff0" }
|
2021-02-26 02:09:13 +03:00
|
|
|
enumset = { git = "https://github.com/danobi/enumset", rev = "4c01c583c27a725948fededbfb3461c572a669a4" }
|
2021-03-04 17:40:30 +03:00
|
|
|
gotham-02 = { package = "gotham", git = "https://github.com/krallin/gotham-02.git", rev = "1eb3b976c31e7e4334b188f3abfa5cc2e5cae033" }
|
|
|
|
gotham_derive-02 = { package = "gotham_derive", git = "https://github.com/krallin/gotham-02.git", rev = "1eb3b976c31e7e4334b188f3abfa5cc2e5cae033" }
|
2021-03-04 17:40:30 +03:00
|
|
|
hyper-02 = { package = "hyper", version = "0.13.10", git = "https://github.com/krallin/hyper-02.git", rev = "9c9453c5ae3e92de4ba30edd81bb45371b8fa8d6" }
|
2020-11-21 03:58:47 +03:00
|
|
|
lru-disk-cache = { git = "https://github.com/mozilla/sccache", rev = "033ebaae69beeb0ac04e8c35d6ff1103487bd9a3" }
|
2021-03-03 15:06:49 +03:00
|
|
|
openssl = { git = "https://github.com/sfackler/rust-openssl", rev = "68fc8ba890d77986b06ca5ce29d1089285fbbcf9" }
|
|
|
|
openssl-sys = { git = "https://github.com/sfackler/rust-openssl", rev = "68fc8ba890d77986b06ca5ce29d1089285fbbcf9" }
|
2021-03-31 12:32:58 +03:00
|
|
|
petgraph = { git = "https://github.com/jkeljo/petgraph", rev = "e3e9dd8632d23973fdc0b42c1117d5e5fc5fa384" }
|
2020-06-17 16:26:05 +03:00
|
|
|
prost = { git = "https://github.com/gabrielrussoc/prost", branch = "protoc-runtime" }
|
|
|
|
prost-build = { git = "https://github.com/gabrielrussoc/prost", branch = "protoc-runtime" }
|
|
|
|
prost-derive = { git = "https://github.com/gabrielrussoc/prost", branch = "protoc-runtime" }
|
|
|
|
prost-types = { git = "https://github.com/gabrielrussoc/prost", branch = "protoc-runtime" }
|
|
|
|
r2d2_sqlite = { git = "https://github.com/jsgf/r2d2-sqlite.git", rev = "6d77a828ca0a3c507a3f58561532a1b6c66c7918" }
|
|
|
|
rustfilt = { git = "https://github.com/jsgf/rustfilt.git", rev = "8141fa7f1caee562ee8daffb2ddeca3d1f0d36e5" }
|
2021-02-26 02:09:13 +03:00
|
|
|
tokio-02 = { package = "tokio", version = "0.2.25", git = "https://github.com/krallin/tokio.git", rev = "818f943db36e32b7c928351c32fe69ac913409f9" }
|
|
|
|
tokio-compat = { package = "tokio-compat", version = "0.1.6", git = "https://github.com/krallin/tokio-compat.git", rev = "c2330c2a0141353e467cc5861988daa2eae7cb54" }
|
2020-06-17 16:26:05 +03:00
|
|
|
tokio-core = { git = "https://github.com/bolinfest/tokio-core", rev = "5f37aa3c627d56ee49154bc851d6930f5ab4398f" }
|
2020-12-17 18:43:34 +03:00
|
|
|
toml = { git = "https://github.com/jsgf/toml-rs", branch = "dotted-table-0.5.7" }
|
2021-02-26 02:09:13 +03:00
|
|
|
|
|
|
|
[workspace]
|
|
|
|
members = [
|
|
|
|
".",
|
|
|
|
"alpn",
|
2021-03-11 15:25:00 +03:00
|
|
|
"benchmarks/derived_data",
|
2021-03-11 15:25:00 +03:00
|
|
|
"benchmarks/simulated_repo",
|
2021-02-26 02:09:13 +03:00
|
|
|
"blobimport_lib",
|
|
|
|
"blobimport_lib/consts",
|
|
|
|
"blobrepo",
|
|
|
|
"blobrepo/blobrepo_hg",
|
|
|
|
"blobrepo/blobsync",
|
|
|
|
"blobrepo/changeset_fetcher",
|
|
|
|
"blobrepo/common",
|
|
|
|
"blobrepo/errors",
|
|
|
|
"blobrepo/override",
|
|
|
|
"blobrepo/repo_blobstore",
|
|
|
|
"blobrepo_utils",
|
|
|
|
"blobstore",
|
|
|
|
"blobstore/blobstore_stats",
|
|
|
|
"blobstore/cacheblob",
|
|
|
|
"blobstore/chaosblob",
|
|
|
|
"blobstore/delayblob",
|
|
|
|
"blobstore/factory",
|
|
|
|
"blobstore/fileblob",
|
|
|
|
"blobstore/if",
|
|
|
|
"blobstore/logblob",
|
|
|
|
"blobstore/memblob",
|
|
|
|
"blobstore/multiplexedblob",
|
|
|
|
"blobstore/packblob",
|
|
|
|
"blobstore/packblob/if",
|
|
|
|
"blobstore/prefixblob",
|
|
|
|
"blobstore/readonlyblob",
|
|
|
|
"blobstore/redactedblobstore",
|
|
|
|
"blobstore/samplingblob",
|
|
|
|
"blobstore/sqlblob",
|
|
|
|
"blobstore/throttledblob",
|
|
|
|
"blobstore/virtually_sharded_blobstore",
|
|
|
|
"blobstore_sync_queue",
|
|
|
|
"bonsai_git_mapping",
|
|
|
|
"bonsai_git_mapping/git_mapping_pushrebase_hook",
|
|
|
|
"bonsai_globalrev_mapping",
|
|
|
|
"bonsai_globalrev_mapping/globalrev_pushrebase_hook",
|
|
|
|
"bonsai_globalrev_mapping/if",
|
|
|
|
"bonsai_hg_mapping",
|
|
|
|
"bonsai_hg_mapping/if",
|
|
|
|
"bonsai_svnrev_mapping",
|
|
|
|
"bonsai_svnrev_mapping/if",
|
|
|
|
"bookmarks",
|
|
|
|
"bookmarks/bookmarks_movement",
|
|
|
|
"bookmarks/bookmarks_types",
|
|
|
|
"bookmarks/dbbookmarks",
|
|
|
|
"bookmarks/warm_bookmarks_cache",
|
|
|
|
"bulkops",
|
|
|
|
"bulkops/bench",
|
|
|
|
"cache_warmup",
|
|
|
|
"changesets",
|
|
|
|
"changesets/if",
|
|
|
|
"cmdlib",
|
mononoke: use MononokeEnvironment in RepoFactory
Summary:
There is a very frustrating operation that happens often when working on the
Mononoke code base:
- You want to add a flag
- You want to consume it in the repo somewhere
Unfortunately, when we need to do this, we end up having to thread this from a
million places and parse it out in every single main() we have.
This is a mess, and it results in every single Mononoke binary starting with
heaps of useless boilerplate:
```
let matches = app.get_matches();
let (caching, logger, mut runtime) = matches.init_mononoke(fb)?;
let config_store = args::init_config_store(fb, &logger, &matches)?;
let mysql_options = args::parse_mysql_options(&matches);
let blobstore_options = args::parse_blobstore_options(&matches)?;
let readonly_storage = args::parse_readonly_storage(&matches);
```
So, this diff updates us to just use MononokeEnvironment directly in
RepoFactory, which means none of that has to happen: we can now add a flag,
parse it into MononokeEnvironment, and get going.
While we're at it, we can also remove blobstore options and all that jazz from
MononokeApiEnvironment since now it's there in the underlying RepoFactory.
Reviewed By: HarveyHunt
Differential Revision: D27767700
fbshipit-source-id: e1e359bf403b4d3d7b36e5f670aa1a7dd4f1d209
2021-04-16 20:26:03 +03:00
|
|
|
"cmdlib/environment",
|
2021-02-26 02:09:13 +03:00
|
|
|
"cmdlib/x_repo",
|
2021-04-27 18:53:40 +03:00
|
|
|
"cmds/copy_blobstore_keys",
|
2021-02-26 02:09:13 +03:00
|
|
|
"commit_rewriting/backsyncer",
|
|
|
|
"commit_rewriting/bookmark_renaming",
|
|
|
|
"commit_rewriting/bookmarks_validator",
|
|
|
|
"commit_rewriting/commit_validator",
|
|
|
|
"commit_rewriting/cross_repo_sync",
|
|
|
|
"commit_rewriting/cross_repo_sync/test_utils",
|
|
|
|
"commit_rewriting/live_commit_sync_config",
|
|
|
|
"commit_rewriting/megarepo",
|
|
|
|
"commit_rewriting/mononoke_x_repo_sync_job",
|
|
|
|
"commit_rewriting/movers",
|
|
|
|
"commit_rewriting/synced_commit_mapping",
|
|
|
|
"common/allocation_tracing",
|
|
|
|
"common/async_limiter",
|
|
|
|
"common/async_limiter/examples/tokio_v2",
|
|
|
|
"common/bounded_traversal",
|
|
|
|
"common/copy_utils",
|
|
|
|
"common/dedupmap",
|
|
|
|
"common/futures_watchdog",
|
|
|
|
"common/iterhelpers",
|
mononoke/rendezvous: introduce query batching
Summary:
This introduces a basic building block for query batching. I called this
rendezvous, since it's about multiple queries meeting up in the same place :)
There are a few (somewhat conflicting) goals this tries to satisfy, so let's go
over them:
1), we'd like to reduce the total number of queries made by batch jobs. For
example, group hg bonsai lookups made by the walker. Those jobs are
characterized by the fact that they have a lot of queries to make, all the
time. Here's an example: https://fburl.com/ods/zuiep7yh.
2), we'd like to reduce the overall number of connections held to MySQL by
our tasks. The main way we achieve this is by reducing the maximum number of
concurrent queries. Indeed, a high total number of queries doesn't necessarily
result in a lot of connections as long as they're not concurrent, because we
can reuse connections. On the other hand, if you dispatch 100 concurrent
queries, that _does_ use 100 connections. This is something that applies to
batch jobs due to their query volume, but also to "interactive" jobs like
Mononoke Server or SCS, just not all the time. Here's an example:
https://fburl.com/ods/o6gp07qp (you can see the query count is overall low, but
sometimes spikes substantially).
2.1) It's also worth noting that concurrent queries are often the result of
many clients wanting the same data, so deduplication is also useful here.
3), we also don't want to impact the latency of interactive jobs when they
need to a little query here or there (i.e. it's largely fine if our jobs all
hold a few connections to MySQL and use them somewhat consistently).
4), we'd like this to make it easier to do batching right. For example, if
you have 100 Bonsais to map to hg, you should be able to just map and call
`future::try_join_all` and have that do the right thing.
5), we don't want "bad" queries to affect other queries negatively. One
example would be the occasional queries we make to Bonsai <-> Hg mapping in
`known` for thousands (if not more) of rows.
6), we want this to be easy to incorporate into the codebase.
So, how do we try to address all of this? Here's how:
- We ... do batching, and we deduplicate requests in a batch. This is the
easier bit and should address #1, #2 and #2.1, #4.
- However, batching is conditional. We notably don't batch very large requests
with the rest (addresses #5). We also don't batch small queries all the time:
we only batch if we are observing a throughput of queries that suggests we
can find some benefit in batching (this targets #3).
- Finally, we have some utilities for common cases like having to group by repo
id (this is `MultiRendezVous`), and this is all configurable via tunables
(and the default is to not do anything).
Reviewed By: StanislavGlebik
Differential Revision: D27010317
fbshipit-source-id: 4a2397255f9785c6722c02e4d419438fd0aafa07
2021-03-19 18:49:00 +03:00
|
|
|
"common/rendezvous",
|
2021-02-26 02:09:13 +03:00
|
|
|
"common/rust/caching_ext",
|
|
|
|
"common/rust/slog_ext",
|
|
|
|
"common/rust/sql_ext",
|
|
|
|
"common/scribe_ext",
|
|
|
|
"common/scuba_ext",
|
|
|
|
"common/sql_construct",
|
2021-04-12 15:21:39 +03:00
|
|
|
"common/timeseries",
|
2021-02-26 02:09:13 +03:00
|
|
|
"common/topo_sort",
|
|
|
|
"common/type_map",
|
|
|
|
"common/uniqueheap",
|
|
|
|
"derived_data",
|
|
|
|
"derived_data/blame",
|
|
|
|
"derived_data/changeset_info",
|
|
|
|
"derived_data/changeset_info/if",
|
|
|
|
"derived_data/deleted_files_manifest",
|
|
|
|
"derived_data/fastlog",
|
|
|
|
"derived_data/filenodes",
|
|
|
|
"derived_data/fsnodes",
|
|
|
|
"derived_data/mercurial_derived_data",
|
|
|
|
"derived_data/skeleton_manifest",
|
|
|
|
"derived_data/test_utils",
|
|
|
|
"derived_data/unodes",
|
|
|
|
"derived_data/utils",
|
|
|
|
"edenapi_server",
|
|
|
|
"edenapi_service",
|
|
|
|
"fastreplay",
|
|
|
|
"filenodes",
|
|
|
|
"filenodes/if",
|
|
|
|
"filestore",
|
|
|
|
"git/git-pool",
|
|
|
|
"git/git_types",
|
|
|
|
"git/git_types/if",
|
|
|
|
"git/gitimport",
|
|
|
|
"git/import_tools",
|
|
|
|
"gotham_ext",
|
|
|
|
"hgcli",
|
|
|
|
"hgproto",
|
|
|
|
"hook_tailer",
|
|
|
|
"hooks",
|
|
|
|
"hooks/content-stores",
|
|
|
|
"hooks/hook_manager_factory",
|
|
|
|
"lfs_import_lib",
|
|
|
|
"lfs_protocol",
|
|
|
|
"lfs_server",
|
|
|
|
"load_limiter",
|
|
|
|
"manifest",
|
2021-04-19 18:33:00 +03:00
|
|
|
"megarepo_api/configo_client",
|
2021-04-22 12:12:40 +03:00
|
|
|
"megarepo_api/megarepo_config",
|
2021-04-21 15:00:58 +03:00
|
|
|
"megarepo_api/megarepo_error",
|
2021-02-26 02:09:13 +03:00
|
|
|
"mercurial/bundle_replay_data",
|
|
|
|
"mercurial/bundles",
|
|
|
|
"mercurial/mutation",
|
|
|
|
"mercurial/revlog",
|
|
|
|
"mercurial/types",
|
|
|
|
"mercurial/types/if",
|
|
|
|
"mercurial/types/mocks",
|
|
|
|
"metaconfig/parser",
|
|
|
|
"metaconfig/types",
|
|
|
|
"microwave",
|
|
|
|
"microwave/builder",
|
|
|
|
"microwave/if",
|
|
|
|
"mononoke_api",
|
|
|
|
"mononoke_api_hg",
|
|
|
|
"mononoke_hg_sync_job",
|
|
|
|
"mononoke_types",
|
|
|
|
"mononoke_types/if",
|
|
|
|
"mononoke_types/mocks",
|
|
|
|
"mutable_counters",
|
|
|
|
"newfilenodes",
|
|
|
|
"observability",
|
|
|
|
"permission_checker",
|
|
|
|
"phases",
|
|
|
|
"pushrebase",
|
2021-04-27 13:51:52 +03:00
|
|
|
"pushrebase/pushrebase_hook",
|
2021-04-27 13:51:52 +03:00
|
|
|
"pushrebase_mutation_mapping",
|
2021-04-16 12:25:01 +03:00
|
|
|
"quiet_stream",
|
2021-02-26 02:09:13 +03:00
|
|
|
"reachabilityindex",
|
|
|
|
"reachabilityindex/common",
|
|
|
|
"reachabilityindex/if",
|
|
|
|
"reachabilityindex/skiplist",
|
|
|
|
"reachabilityindex/test-helpers",
|
|
|
|
"regenerate_hg_filenodes",
|
2021-03-22 17:23:06 +03:00
|
|
|
"repo_attributes/repo_derived_data",
|
2021-03-22 17:23:06 +03:00
|
|
|
"repo_attributes/repo_identity",
|
2021-02-26 02:09:13 +03:00
|
|
|
"repo_client",
|
|
|
|
"repo_client/getbundle_response",
|
|
|
|
"repo_client/mononoke_repo",
|
|
|
|
"repo_client/obsolete",
|
|
|
|
"repo_client/remotefilelog",
|
|
|
|
"repo_client/repo_read_write_status",
|
|
|
|
"repo_client/reverse_filler_queue",
|
|
|
|
"repo_client/scribe_commit_queue",
|
|
|
|
"repo_client/streaming_clone",
|
|
|
|
"repo_client/unbundle",
|
|
|
|
"repo_client/wirepack",
|
2021-04-07 23:59:27 +03:00
|
|
|
"repo_factory",
|
2021-03-25 17:32:21 +03:00
|
|
|
"repo_factory/test_repo_factory",
|
2021-02-26 02:09:13 +03:00
|
|
|
"repo_import",
|
|
|
|
"revset",
|
|
|
|
"revset/revset-test-helper",
|
|
|
|
"segmented_changelog",
|
|
|
|
"segmented_changelog/bench/concurrent_idmap",
|
2021-03-22 17:23:06 +03:00
|
|
|
"segmented_changelog/types",
|
2021-02-26 02:09:13 +03:00
|
|
|
"server",
|
|
|
|
"server/context",
|
|
|
|
"server/monitoring",
|
2021-03-22 20:10:53 +03:00
|
|
|
"server/qps",
|
2021-02-26 02:09:13 +03:00
|
|
|
"server/repo_listener",
|
|
|
|
"server/session_id",
|
|
|
|
"sshrelay",
|
2021-03-13 01:45:03 +03:00
|
|
|
"streaming_clone",
|
2021-02-26 02:09:13 +03:00
|
|
|
"tests/fixtures",
|
|
|
|
"tests/utils",
|
|
|
|
"tests/write_stub_log_entry",
|
|
|
|
"time_window_counter",
|
|
|
|
"tunables",
|
|
|
|
"tunables/tunables-derive",
|
|
|
|
"unbundle_replay",
|
|
|
|
"walker",
|
|
|
|
]
|