facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-10 08:47:12 +03:00
Code Issues Projects Releases Wiki Activity
09cd8a3058
sapling/eden/mononoke/Cargo.toml
Thomas Orozco 07907b2b26 mononoke/virtually_sharded_blobstore: merge in the context_concurrency_blobstore 
Summary:
There is inevitably interaction between caching, deduplication and rate
limiting:

- You don't want the rate limiting to be above caching (in the blobstore stack,
  that is), because you shouldn't rate limits cache hits (this is where we are
  today).
- You don't want the rate limiting to below deduplication, because then you get
  priority inversion where a low-priority rate-limited request might hold the
  semaphore while a higher-priority, non rate limited request wants to do the
  same fetch (we could have moved rate limiting here prior to introducing
  deduplication, but I didn't do it earlier because I wanted to eventually
  introduce deduplication).

So, now that we have caching and deduplication in the same blobstore, let's
also incorporate rate limiting there!.

Note that this also brings a potential motivation for moving Memcache into this
blobstore, in case we don't want rate limiting to apply to requests before they
go to the _actual_ blobstore (I did not do this in this diff).

The design here when accessing the blobstore is as follows:

- Get the semaphore
- Check if the data is in cache, if so release the semaphore and return the
  data.
- Otherwise, check if we are rater limited.

Then, if we are rate limited:

- Release the semaphore
- Wait for our turn
- Acquire the semaphore again
- Check the cache again (someone might have put the data we want while we were
  waiting).
    - If the data is there, then return our rate limit token.
    - If the data isn't there, then proceed to query the blobstore.

If we aren't rate limited, then we just proceed to query the blobstore.

There are a couple subtle aspects of this:

- If we have a "late" cache hit (i.e. after we waited for rate limiting), then
  we'll have waited but we won't need to query the blobstore.
    - This is important when a large number of requests from the same key
      arrive at the same time and get rate limited. If we don't do this second
      cache check or if we don't return the token, then we'll consume a rate
      limiting token for each request (instead of 1 for the first request).
- If a piece of data isn't cacheable, we should treat it like a cache hit with
  regard to semaphores (i.e. release early), but like a miss with regard to
  rate limits (i.e. wait).

Both of those are addressed captured in the code by returning the `Ticket` on a
cache hit. We can then choose to either return the ticket on a cache hit, or wait
for it on a cache miss.

(all of this logic is captured in unit tests, we can remove any of the blocks
there in `Shards::acquire` and a test will fail)

Reviewed By: farnz

Differential Revision: D22374606

fbshipit-source-id: c3a48805d3cdfed2a885bec8c47c173ee7ebfe2d
2020-07-06 04:38:31 -07:00

302 lines
9.5 KiB
TOML
Raw Blame History

[package]
name = "eden_mononoke"
edition = "2018"
version = "0.1.0"
authors = ['Facebook']
license = "GPLv2+"
include = ["cmds/aliasverify.rs", "cmds/backfill_derived_data/**/*.rs", "cmds/backfill_git_mapping.rs", "cmds/benchmark_storage_config/**/*.rs", "cmds/bonsai_verify/**/*.rs", "cmds/configlint.rs", "cmds/dumprev.rs", "cmds/idxdump.rs", "cmds/lfs_import.rs", "cmds/rechunker.rs", "cmds/revlogrepo.rs", "cmds/statistics_collector.rs", "cmds/upload_globalrevs.rs"]
[[bin]]
name = "aliasverify"
path = "cmds/aliasverify.rs"
[[bin]]
name = "backfill_derived_data"
path = "cmds/backfill_derived_data/main.rs"
[[bin]]
name = "backfill_git_mapping"
path = "cmds/backfill_git_mapping.rs"
[[bin]]
name = "benchmark_storage_config"
path = "cmds/benchmark_storage_config/main.rs"
[[bin]]
name = "bonsai_verify"
path = "cmds/bonsai_verify/main.rs"
[[bin]]
name = "configlint"
path = "cmds/configlint.rs"
[[bin]]
name = "dumprev"
path = "cmds/dumprev.rs"
[[bin]]
name = "idxdump"
path = "cmds/idxdump.rs"
[[bin]]
name = "lfs_import"
path = "cmds/lfs_import.rs"
[[bin]]
name = "rechunker"
path = "cmds/rechunker.rs"
[[bin]]
name = "revlogrepo"
path = "cmds/revlogrepo.rs"
[[bin]]
name = "statistics_collector"
path = "cmds/statistics_collector.rs"
[[bin]]
name = "upload_globalrevs"
path = "cmds/upload_globalrevs.rs"
[dependencies]
blame = { path = "derived_data/blame" }
blobrepo = { path = "blobrepo" }
blobrepo_factory = { path = "blobrepo/factory" }
blobrepo_hg = { path = "blobrepo/blobrepo_hg" }
blobrepo_override = { path = "blobrepo/override" }
blobrepo_utils = { path = "blobrepo_utils" }
blobstore = { path = "blobstore" }
blobstore_factory = { path = "blobstore/factory" }
bonsai_globalrev_mapping = { path = "bonsai_globalrev_mapping" }
bookmarks = { path = "bookmarks" }
bulkops = { path = "bulkops" }
cacheblob = { path = "blobstore/cacheblob" }
changesets = { path = "changesets" }
cmdlib = { path = "cmdlib" }
context = { path = "server/context" }
deleted_files_manifest = { path = "derived_data/deleted_files_manifest" }
derived_data = { path = "derived_data" }
derived_data_utils = { path = "derived_data/utils" }
fastlog = { path = "derived_data/fastlog" }
filestore = { path = "filestore" }
fsnodes = { path = "derived_data/fsnodes" }
lfs_import_lib = { path = "lfs_import_lib" }
manifest = { path = "manifest" }
mercurial_revlog = { path = "mercurial/revlog" }
mercurial_types = { path = "mercurial/types" }
metaconfig_types = { path = "metaconfig/types" }
mononoke_types = { path = "mononoke_types" }
revset = { path = "revset" }
scuba_ext = { path = "common/scuba_ext" }
unodes = { path = "derived_data/unodes" }
cloned = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
failure_ext = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
fbinit = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
futures_ext = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
futures_stats = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
lock_ext = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
stats = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
time_ext = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
anyhow = "1.0"
ascii = "1.0"
async-trait = "0.1.29"
bytes = { version = "0.5", features = ["serde"] }
clap = "2.33"
criterion = "0.3"
futures = { version = "0.3", features = ["async-await", "compat"] }
futures-old = { package = "futures", version = "0.1" }
futures-util = "0.3"
itertools = "0.8"
rand = { version = "0.7", features = ["small_rng"] }
serde = { version = "1.0", features = ["derive", "rc"] }
serde_derive = "1.0"
sha2 = "0.8"
slog = { version = "2.5", features = ["max_level_debug"] }
tokio = { version = "=0.2.13", features = ["full"] }
tokio-compat = "0.1"
toml = "=0.5.6"
[dev-dependencies]
fixtures = { path = "tests/fixtures" }
tests_utils = { path = "tests/utils" }
maplit = "1.0"
[workspace]
members = [
".",
"backsyncer",
"benchmark",
"blobimport_lib",
"blobrepo",
"blobrepo/blobrepo_hg",
"blobrepo/blobsync",
"blobrepo/changeset_fetcher",
"blobrepo/errors",
"blobrepo/factory",
"blobrepo/override",
"blobrepo/repo_blobstore",
"blobrepo_utils",
"blobstore",
"blobstore/blobstore_stats",
"blobstore/cacheblob",
"blobstore/chaosblob",
"blobstore/delayblob",
"blobstore/factory",
"blobstore/fileblob",
"blobstore/if",
"blobstore/logblob",
"blobstore/memblob",
"blobstore/multiplexedblob",
"blobstore/packblob",
"blobstore/packblob/if",
"blobstore/prefixblob",
"blobstore/readonlyblob",
"blobstore/redactedblobstore",
"blobstore/samplingblob",
"blobstore/sqlblob",
"blobstore/throttledblob",
"blobstore/virtually_sharded_blobstore",
"blobstore_sync_queue",
"bonsai_git_mapping",
"bonsai_git_mapping/git_mapping_pushrebase_hook",
"bonsai_globalrev_mapping",
"bonsai_globalrev_mapping/globalrev_pushrebase_hook",
"bonsai_hg_mapping",
"bonsai_hg_mapping/if",
"bookmarks",
"bookmarks/bookmarks_types",
"bookmarks/dbbookmarks",
"bookmarks/warm_bookmarks_cache",
"bulkops",
"cache_warmup",
"changesets",
"changesets/if",
"cmdlib",
"cmdlib/x_repo",
"cmds/blobrepo_checker",
"commit_rewriting/bookmark_renaming",
"commit_rewriting/cross_repo_sync",
"commit_rewriting/cross_repo_sync/test_utils",
"commit_rewriting/live_commit_sync_config",
"commit_rewriting/movers",
"commit_rewriting/synced_commit_mapping",
"common/allocation_tracing",
"common/async_limiter",
"common/async_limiter/examples/tokio_v2",
"common/bounded_traversal",
"common/dedupmap",
"common/rust/caching_ext",
"common/rust/slog_ext",
"common/rust/sql_ext",
"common/scribe_ext",
"common/scuba_ext",
"common/sql_construct",
"common/topo_sort",
"common/type_map",
"common/uniqueheap",
"derived_data",
"derived_data/blame",
"derived_data/changeset_info",
"derived_data/changeset_info/if",
"derived_data/deleted_files_manifest",
"derived_data/fastlog",
"derived_data/filenodes",
"derived_data/fsnodes",
"derived_data/mercurial_derived_data",
"derived_data/unodes",
"derived_data/utils",
"edenapi_server",
"fastreplay",
"filenodes",
"filenodes/if",
"filestore",
"git/git_types",
"git/git_types/if",
"git/gitimport",
"git/import_tools",
"gotham_ext",
"hgcli",
"hgproto",
"hook_tailer",
"hooks",
"hooks/content-stores",
"lfs_import_lib",
"lfs_protocol",
"lfs_server",
"load_limiter",
"manifest",
"manifest/test_utils",
"megarepolib",
"mercurial/bundles",
"mercurial/mutation",
"mercurial/revlog",
"mercurial/types",
"mercurial/types/if",
"mercurial/types/mocks",
"metaconfig/parser",
"metaconfig/types",
"microwave",
"microwave/builder",
"microwave/if",
"mononoke_api",
"mononoke_types",
"mononoke_types/if",
"mononoke_types/mocks",
"mutable_counters",
"newfilenodes",
"permission_checker",
"phases",
"pushrebase",
"reachabilityindex",
"reachabilityindex/common",
"reachabilityindex/if",
"reachabilityindex/skiplist",
"reachabilityindex/test-helpers",
"repo_client",
"repo_client/getbundle_response",
"repo_client/mononoke_repo",
"repo_client/obsolete",
"repo_client/remotefilelog",
"repo_client/repo_read_write_status",
"repo_client/reverse_filler_queue",
"repo_client/scribe_commit_queue",
"repo_client/streaming_clone",
"repo_client/unbundle",
"repo_client/wirepack",
"repo_import",
"revset",
"revset/revset-test-helper",
"segmented_changelog",
"server",
"server/context",
"server/monitoring",
"server/repo_listener",
"server/session_id",
"sshrelay",
"tests/fixtures",
"tests/utils",
"tests/write_stub_log_entry",
"time_window_counter",
"tunables",
"tunables/tunables-derive",
"unbundle_replay",
"walker",
]
[patch.crates-io]
chashmap = { git = "https://gitlab.redox-os.org/ahornby/chashmap", rev = "901ace2ca3cdbc2095adb1af111d211e254e2aae" }
const-random = { git = "https://github.com/fbsource/const-random", rev = "374c5b46427fe2ffbf6acbd9c1687e0f1a809f95" }
gotham = { git = "https://github.com/krallin/gotham.git", branch = "socket_data" }
gotham_derive = { git = "https://github.com/krallin/gotham.git", branch = "socket_data" }
prost = { git = "https://github.com/gabrielrussoc/prost", branch = "protoc-runtime" }
prost-build = { git = "https://github.com/gabrielrussoc/prost", branch = "protoc-runtime" }
prost-derive = { git = "https://github.com/gabrielrussoc/prost", branch = "protoc-runtime" }
prost-types = { git = "https://github.com/gabrielrussoc/prost", branch = "protoc-runtime" }
protobuf-build = { git = "https://github.com/tikv/protobuf-build.git", rev = "1b8b7f8b4aa1be4647acd85bab9345583a5df4f1" }
pyo3 = { git = "https://github.com/PyO3/pyo3.git", rev = "4af61e83c34a04d74679a1048ee3866831a327ab" }
r2d2_sqlite = { git = "https://github.com/jsgf/r2d2-sqlite.git", rev = "6d77a828ca0a3c507a3f58561532a1b6c66c7918" }
rustfilt = { git = "https://github.com/jsgf/rustfilt.git", rev = "8141fa7f1caee562ee8daffb2ddeca3d1f0d36e5" }
tokio-core = { git = "https://github.com/bolinfest/tokio-core", rev = "5f37aa3c627d56ee49154bc851d6930f5ab4398f" }
toml = { git = "https://github.com/jsgf/toml-rs", branch = "master" }
Reference in New Issue View Git Blame Copy Permalink