sapling/eden/mononoke/Cargo.toml

414 lines
16 KiB
TOML
Raw Normal View History

[package]
name = "eden_mononoke"
version = "0.1.0"
authors = ["Facebook"]
edition = "2018"
license = "GPLv2+"
[[bin]]
name = "admin"
path = "cmds/admin/main.rs"
[[bin]]
name = "aliasverify"
path = "cmds/aliasverify.rs"
[[bin]]
name = "backfill_derived_data"
path = "cmds/backfill_derived_data/main.rs"
[[bin]]
name = "backfill_mapping"
path = "cmds/backfill_mapping.rs"
[[bin]]
name = "benchmark_filestore"
path = "cmds/benchmark_filestore.rs"
[[bin]]
name = "benchmark_storage_config"
path = "cmds/benchmark_storage_config/main.rs"
[[bin]]
name = "blobimport"
path = "cmds/blobimport.rs"
[[bin]]
name = "blobstore_healer"
path = "cmds/blobstore_healer/main.rs"
[[bin]]
name = "bonsai_verify"
path = "cmds/bonsai_verify/main.rs"
[[bin]]
name = "configlint"
path = "cmds/configlint.rs"
[[bin]]
name = "dumprev"
path = "cmds/dumprev.rs"
[[bin]]
name = "idxdump"
path = "cmds/idxdump.rs"
[[bin]]
name = "lfs_import"
path = "cmds/lfs_import.rs"
[[bin]]
name = "manual_scrub"
path = "cmds/manual_scrub/main.rs"
[[bin]]
name = "rechunker"
path = "cmds/rechunker.rs"
[[bin]]
name = "revlogrepo"
path = "cmds/revlogrepo.rs"
[[bin]]
name = "segmented_changelog_seeder"
path = "cmds/segmented_changelog_seeder.rs"
[[bin]]
name = "segmented_changelog_tailer"
path = "cmds/segmented_changelog_tailer.rs"
[[bin]]
name = "sqlblob_gc"
path = "cmds/sqlblob_gc/main.rs"
[[bin]]
name = "statistics_collector"
path = "cmds/statistics_collector.rs"
[[bin]]
name = "streaming_clone_warmup"
path = "cmds/streaming_clone_warmup/main.rs"
[[bin]]
name = "upload_globalrevs"
path = "cmds/upload_globalrevs.rs"
[dependencies]
anyhow = "1.0"
ascii = "1.0"
async-trait = "0.1.29"
backsyncer = { path = "commit_rewriting/backsyncer", version = "0.1.0" }
blame = { path = "derived_data/blame", version = "0.1.0" }
blobimport_lib = { path = "blobimport_lib", version = "0.1.0" }
blobrepo = { path = "blobrepo", version = "0.1.0" }
blobrepo_factory = { path = "blobrepo/factory", version = "0.1.0" }
blobrepo_hg = { path = "blobrepo/blobrepo_hg", version = "0.1.0" }
blobrepo_override = { path = "blobrepo/override", version = "0.1.0" }
blobrepo_utils = { path = "blobrepo_utils", version = "0.1.0" }
blobstore = { path = "blobstore", version = "0.1.0" }
blobstore_factory = { path = "blobstore/factory", version = "0.1.0" }
blobstore_sync_queue = { path = "blobstore_sync_queue", version = "0.1.0" }
bonsai_globalrev_mapping = { path = "bonsai_globalrev_mapping", version = "0.1.0" }
bookmark_renaming = { path = "commit_rewriting/bookmark_renaming", version = "0.1.0" }
bookmarks = { path = "bookmarks", version = "0.1.0" }
bookmarks_movement = { path = "bookmarks/bookmarks_movement", version = "0.1.0" }
borrowed = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master", version = "0.1.0" }
bounded_traversal = { path = "common/bounded_traversal", version = "0.1.0" }
bulkops = { path = "bulkops", version = "0.1.0" }
bytes = { version = "0.5", features = ["serde"] }
bytesize = "1.0"
cacheblob = { path = "blobstore/cacheblob", version = "0.1.0" }
cached_config = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master", version = "0.1.0" }
cachelib = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master", version = "0.1.0" }
changeset_fetcher = { path = "blobrepo/changeset_fetcher", version = "0.1.0" }
changesets = { path = "changesets", version = "0.1.0" }
chrono = { version = "0.4", features = ["serde"] }
clap = "2.33"
cloned = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master", version = "0.1.0" }
cmdlib = { path = "cmdlib", version = "0.1.0" }
context = { path = "server/context", version = "0.1.0" }
copy_utils = { path = "common/copy_utils", version = "0.1.0" }
criterion = "=0.3.1"
cross_repo_sync = { path = "commit_rewriting/cross_repo_sync", version = "0.1.0" }
dbbookmarks = { path = "bookmarks/dbbookmarks", version = "0.1.0" }
deleted_files_manifest = { path = "derived_data/deleted_files_manifest", version = "0.1.0" }
derived_data = { path = "derived_data", version = "0.1.0" }
derived_data_filenodes = { path = "derived_data/filenodes", version = "0.1.0" }
derived_data_utils = { path = "derived_data/utils", version = "0.1.0" }
failure_ext = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master", version = "0.1.0" }
fastlog = { path = "derived_data/fastlog", version = "0.1.0" }
fbinit = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master", version = "0.1.0" }
fbthrift = { git = "https://github.com/facebook/fbthrift.git", branch = "master", version = "0.0.1+unstable" }
filenodes = { path = "filenodes", version = "0.1.0" }
filestore = { path = "filestore", version = "0.1.0" }
fsnodes = { path = "derived_data/fsnodes", version = "0.1.0" }
futures = { version = "0.3.5", features = ["async-await", "compat"] }
futures-old = { package = "futures", version = "0.1.30" }
futures-util = "0.3.7"
futures_03_ext = { package = "futures_ext", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master", version = "0.1.0" }
futures_ext = { package = "futures_01_ext", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master", version = "0.1.0" }
futures_stats = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master", version = "0.1.0" }
git_types = { path = "git/git_types", version = "0.1.0" }
humantime = "1.3"
itertools = "0.8"
lazy_static = "1.0"
lfs_import_lib = { path = "lfs_import_lib", version = "0.1.0" }
live_commit_sync_config = { path = "commit_rewriting/live_commit_sync_config", version = "0.1.0" }
lock_ext = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master", version = "0.1.0" }
manifest = { path = "manifest", version = "0.1.0" }
maplit = "1.0"
memblob = { path = "blobstore/memblob", version = "0.1.0" }
mercurial_bundle_replay_data = { path = "mercurial/bundle_replay_data", version = "0.1.0" }
mercurial_derived_data = { path = "derived_data/mercurial_derived_data", version = "0.1.0" }
mercurial_revlog = { path = "mercurial/revlog", version = "0.1.0" }
mercurial_types = { path = "mercurial/types", version = "0.1.0" }
metaconfig_types = { path = "metaconfig/types", version = "0.1.0" }
mononoke_hg_sync_job_helper_lib = { path = "mononoke_hg_sync_job", version = "0.1.0" }
mononoke_types = { path = "mononoke_types", version = "0.1.0" }
mutable_counters = { path = "mutable_counters", version = "0.1.0" }
prefixblob = { path = "blobstore/prefixblob", version = "0.1.0" }
pushrebase = { path = "pushrebase", version = "0.1.0" }
rand = { version = "0.7", features = ["small_rng"] }
redactedblobstore = { path = "blobstore/redactedblobstore", version = "0.1.0" }
regex = "1.4.2"
revset = { path = "revset", version = "0.1.0" }
scuba_ext = { path = "common/scuba_ext", version = "0.1.0" }
segmented_changelog = { path = "segmented_changelog", version = "0.1.0" }
segmented_changelog: update OnDemandUpdateDag to have smaller critical sections Summary: The on demand update code we have is the most basic logic that we could have. The main problem is that it has long and redundant write locks. This change reduces the write lock strictly to the section that has to update the in memory IdDag. Updating the Dag has 3 phases: * loading the data that is required for the update; * updating the IdMap; * updating the IdDag; The Dag can function well for serving requests as long as the commits involved have been built so we want to have easy read access to both the IdMap and the IdDag. The IdMap is a very simple structure and because it's described as an Arc<dyn IdMap> we push the update locking logic to the storage. The IdDag is a complicated structure that we ask to update itself. Those functions take mutable references. Updating the storage of the iddag to hide the complexities of locking is more difficult. We deal with the IdDag directly by wrapping it in a RwLock. The RwLock allows for easy read access which we expect to be the predominant access pattern. Updates to the dag are not completely stable so racing updates can have conflicting results. In case of conflics one of the update processes would have to restart. It's easier to reason about the process if we just allow one "thread" to start an update process. The update process is locked by a sync mutex. The "threads" that fail the race to update are asked to wait until the ongoing update is complete. The waiters will poll on a shared future that tracks the ongoing dag update. After the update is complete the waiters will go back to checking if the data they have is available in the dag. It is possible that the dag is updated in between determining that the an update is needed and acquiring the ongoing_update lock. This is fine because the update building process checks the state of dag before the dag and updates only what is necessary if necessary. Reviewed By: krallin Differential Revision: D26508430 fbshipit-source-id: cd3bceed7e0ffb00aee64433816b5a23c0508d3c
2021-02-23 05:14:45 +03:00
serde = { version = "=1.0.118", features = ["derive", "rc"] }
serde_derive = "1.0"
serde_json = { version = "1.0", features = ["float_roundtrip"] }
sha2 = "0.8"
skeleton_manifest = { path = "derived_data/skeleton_manifest", version = "0.1.0" }
skiplist = { path = "reachabilityindex/skiplist", version = "0.1.0" }
slog = { version = "2.5", features = ["max_level_debug"] }
sql_construct = { path = "common/sql_construct", version = "0.1.0" }
sql_ext = { path = "common/rust/sql_ext", version = "0.1.0" }
sqlblob = { path = "blobstore/sqlblob", version = "0.1.0" }
stats = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master", version = "0.1.0" }
streaming_clone = { path = "repo_client/streaming_clone", version = "0.1.0" }
synced_commit_mapping = { path = "commit_rewriting/synced_commit_mapping", version = "0.1.0" }
thiserror = "1.0"
throttledblob = { path = "blobstore/throttledblob", version = "0.1.0" }
time_ext = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master", version = "0.1.0" }
tokio = { version = "0.2.25", features = ["full", "test-util"] }
tokio-util = { version = "0.2", features = ["codec", "udp"] }
tokio_shim = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master", version = "0.1.0" }
toml = "=0.5.7"
unodes = { path = "derived_data/unodes", version = "0.1.0" }
xdiff = { path = "../scm/lib/xdiff", version = "0.1.0" }
[dev-dependencies]
fixtures = { path = "tests/fixtures", version = "0.1.0" }
tests_utils = { path = "tests/utils", version = "0.1.0" }
[workspace]
members = [
".",
"alpn",
"benchmark",
"blobimport_lib",
"blobimport_lib/consts",
"blobrepo",
"blobrepo/blobrepo_hg",
"blobrepo/blobsync",
"blobrepo/changeset_fetcher",
"blobrepo/common",
"blobrepo/errors",
"blobrepo/factory",
"blobrepo/override",
"blobrepo/repo_blobstore",
"blobrepo_utils",
"blobstore",
"blobstore/blobstore_stats",
"blobstore/cacheblob",
"blobstore/chaosblob",
"blobstore/delayblob",
"blobstore/factory",
"blobstore/fileblob",
"blobstore/if",
"blobstore/logblob",
"blobstore/memblob",
"blobstore/multiplexedblob",
"blobstore/packblob",
"blobstore/packblob/if",
"blobstore/prefixblob",
"blobstore/readonlyblob",
"blobstore/redactedblobstore",
"blobstore/samplingblob",
"blobstore/sqlblob",
"blobstore/throttledblob",
"blobstore/virtually_sharded_blobstore",
"blobstore_sync_queue",
"bonsai_git_mapping",
"bonsai_git_mapping/git_mapping_pushrebase_hook",
"bonsai_globalrev_mapping",
"bonsai_globalrev_mapping/globalrev_pushrebase_hook",
"bonsai_globalrev_mapping/if",
"bonsai_hg_mapping",
"bonsai_hg_mapping/if",
"bonsai_svnrev_mapping",
"bonsai_svnrev_mapping/if",
"bookmarks",
"bookmarks/bookmarks_movement",
"bookmarks/bookmarks_types",
"bookmarks/dbbookmarks",
"bookmarks/warm_bookmarks_cache",
"bulkops",
"bulkops/bench",
"cache_warmup",
"changesets",
"changesets/if",
"cmdlib",
"cmdlib/x_repo",
"commit_rewriting/backsyncer",
"commit_rewriting/bookmark_renaming",
"commit_rewriting/bookmarks_validator",
"commit_rewriting/commit_validator",
"commit_rewriting/cross_repo_sync",
"commit_rewriting/cross_repo_sync/test_utils",
"commit_rewriting/live_commit_sync_config",
"commit_rewriting/megarepo",
"commit_rewriting/mononoke_x_repo_sync_job",
"commit_rewriting/movers",
"commit_rewriting/synced_commit_mapping",
"common/allocation_tracing",
"common/async_limiter",
"common/async_limiter/examples/tokio_v2",
"common/bounded_traversal",
"common/copy_utils",
"common/dedupmap",
"common/futures_watchdog",
"common/iterhelpers",
"common/rust/caching_ext",
"common/rust/slog_ext",
"common/rust/sql_ext",
"common/scribe_ext",
"common/scuba_ext",
"common/sql_construct",
"common/topo_sort",
"common/type_map",
"common/uniqueheap",
"derived_data",
"derived_data/blame",
"derived_data/changeset_info",
"derived_data/changeset_info/if",
"derived_data/deleted_files_manifest",
"derived_data/fastlog",
"derived_data/filenodes",
"derived_data/fsnodes",
"derived_data/mercurial_derived_data",
"derived_data/skeleton_manifest",
"derived_data/test_utils",
"derived_data/unodes",
"derived_data/utils",
"edenapi_server",
"edenapi_service",
"fastreplay",
"filenodes",
"filenodes/if",
"filestore",
"git/git-pool",
"git/git_types",
"git/git_types/if",
"git/gitimport",
"git/import_tools",
"gotham_ext",
"hgcli",
"hgproto",
"hook_tailer",
"hooks",
"hooks/content-stores",
"hooks/hook_manager_factory",
"lfs_import_lib",
"lfs_protocol",
"lfs_server",
"load_limiter",
"manifest",
"mercurial/bundle_replay_data",
"mercurial/bundles",
"mercurial/mutation",
"mercurial/revlog",
"mercurial/types",
"mercurial/types/if",
"mercurial/types/mocks",
"metaconfig/parser",
"metaconfig/types",
"microwave",
"microwave/builder",
"microwave/if",
"mononoke_api",
"mononoke_api_hg",
"mononoke_hg_sync_job",
"mononoke_types",
"mononoke_types/if",
"mononoke_types/mocks",
"mutable_counters",
"newfilenodes",
2020-12-05 01:25:24 +03:00
"observability",
"permission_checker",
"phases",
"pushrebase",
"reachabilityindex",
"reachabilityindex/common",
"reachabilityindex/if",
"reachabilityindex/skiplist",
"reachabilityindex/test-helpers",
"regenerate_hg_filenodes",
"repo_client",
"repo_client/getbundle_response",
"repo_client/mononoke_repo",
"repo_client/obsolete",
"repo_client/remotefilelog",
"repo_client/repo_read_write_status",
"repo_client/reverse_filler_queue",
"repo_client/scribe_commit_queue",
"repo_client/streaming_clone",
"repo_client/unbundle",
"repo_client/wirepack",
"repo_import",
"revset",
"revset/revset-test-helper",
"segmented_changelog",
"segmented_changelog/bench/concurrent_idmap",
"server",
"server/context",
"server/monitoring",
"server/repo_listener",
"server/session_id",
"sshrelay",
"tests/fixtures",
"tests/utils",
"tests/write_stub_log_entry",
"time_window_counter",
mononoke: Add tunables - a simple form of config hot reloading Summary: Currently, Mononoke's configs are loaded at startup and only refreshed during restart. There are some exceptions to this, including throttling limits. Other Mononoke services (such as the LFS server) have their own implementations of hot reloadable configs, however there isn't a universally agreed upon method. Static configs makes it hard to roll out features gradually and safely. If a bad config option is enabled, it can't be rectified until the entire tier is restarted. However, Mononoke's code is structured with static configs in mind and doesn't support hot reloading. Changing this would require a lot of work (imagine trying to swap out blobstore configs during run time) and wouldn't necessarily provide much benefit. Instead, add a subset of hot reloadable configs called tunables. Tunables are accessible from anywhere in the code and are cheap to read as they only require reading an atomic value. This means that they can be used even in hot code paths. Currently tunables support reloading boolean values and i64s. In the future, I'd like to expand tunables to include more functionality, such as a rollout percentage. The `--tunables-config` flag points to a configerator spec that exports a Tunables thrift struct. This allows differents tiers and Mononoke services to have their own tunables. If this isn't provided, `MononokeTunables::default()` will be used. This diff adds a proc_macro that will generate the relevant `get` and `update` methods for the fields added to a struct which derives `Tunables`. This struct is then stored in a `once_cell` and can be accessed using `tunables::tunables()`. To add a new tunable, add a field to the `MononokeTunables` struct that is of type `AtomicBool` or `AtomicI64`. Update the relevant tunables configerator config to include your new field, with the exact same name. Removing a tunable from `MononokeTunables` is fine, as is removing a tunable from configerator. If the `--tunables-config` path isn't passed, then a default tunables config located at `scm/mononoke/tunables/default` will be loaded. There is also the `--disable-tunables` flag that won't load anything from configerator, it will instead use the `Tunable` struct's `default()` method to initialise it. This is useful in integration tests. Reviewed By: StanislavGlebik Differential Revision: D21177252 fbshipit-source-id: 02a93c1ceee99066019b23d81ea308e4c565d371
2020-05-01 02:06:10 +03:00
"tunables",
"tunables/tunables-derive",
"unbundle_replay",
"walker",
]
[patch.crates-io]
addr2line = { git ="https://github.com/gimli-rs/addr2line.git", rev = "0b6b6018b5b252a18e628fba03885f7d21844b3c" }
bytecount = { git = "https://github.com/llogiq/bytecount", rev = "469eaf8395c99397cd64d059737a9054aa014088" }
chashmap = { git = "https://gitlab.redox-os.org/ahornby/chashmap", rev = "901ace2ca3cdbc2095adb1af111d211e254e2aae" }
const-random = { git = "https://github.com/fbsource/const-random", rev = "374c5b46427fe2ffbf6acbd9c1687e0f1a809f95" }
gotham = { git = "https://github.com/krallin/gotham.git", rev = "59f7f3fe8958837cc4343c67ee5c2c1f11ce65ab"}
gotham_derive = { git = "https://github.com/krallin/gotham.git", rev = "59f7f3fe8958837cc4343c67ee5c2c1f11ce65ab"}
lru-disk-cache = { git = "https://github.com/mozilla/sccache", rev = "033ebaae69beeb0ac04e8c35d6ff1103487bd9a3" }
openssl = { git = "https://github.com/KapJI/rust-openssl", rev = "e0d808e27fec1818b1eecf362f69764ed860aa34" }
openssl-sys = { git = "https://github.com/KapJI/rust-openssl", rev = "e0d808e27fec1818b1eecf362f69764ed860aa34" }
prost = { git = "https://github.com/gabrielrussoc/prost", branch = "protoc-runtime" }
prost-build = { git = "https://github.com/gabrielrussoc/prost", branch = "protoc-runtime" }
prost-derive = { git = "https://github.com/gabrielrussoc/prost", branch = "protoc-runtime" }
prost-types = { git = "https://github.com/gabrielrussoc/prost", branch = "protoc-runtime" }
r2d2_sqlite = { git = "https://github.com/jsgf/r2d2-sqlite.git", rev = "6d77a828ca0a3c507a3f58561532a1b6c66c7918" }
rustfilt = { git = "https://github.com/jsgf/rustfilt.git", rev = "8141fa7f1caee562ee8daffb2ddeca3d1f0d36e5" }
smallvec = { git = "https://github.com/ahornby/rust-smallvec.git", rev = "708e6f09ced4ed211a7c63f53bce7dfe2bbda478"}
tokio-core = { git = "https://github.com/bolinfest/tokio-core", rev = "5f37aa3c627d56ee49154bc851d6930f5ab4398f" }
toml = { git = "https://github.com/jsgf/toml-rs", branch = "dotted-table-0.5.7" }
enumset = { git = "https://github.com/danobi/enumset", rev = "4c01c583c27a725948fededbfb3461c572a669a4" }
cursive_buffered_backend = { git = "https://github.com/chengxiong-ruan/cursive_buffered_backend", branch = "upgrade_cursive_core_from_v0.4.1"}
hyper-02 = { git = "https://github.com/krallin/hyper.git", rev = "9c9453c5ae3e92de4ba30edd81bb45371b8fa8d6", package = "hyper", version = "0.13.10" }
tokio-02 = { git = "https://github.com/krallin/tokio.git", rev = "818f943db36e32b7c928351c32fe69ac913409f9", package = "tokio", version = "0.2.25" }
tokio-compat = { git = "https://github.com/krallin/tokio-compat.git", rev = "c2330c2a0141353e467cc5861988daa2eae7cb54", package = "tokio-compat", version = "0.1.6" }