facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-10 00:45:18 +03:00
Code Issues Projects Releases Wiki Activity

mononoke: maybe override SessionClass while deriving data

Browse Source 
Summary:
We had issues with mononoke writing too much blobstore sync queue entries while
deriving data for large commits. We've contemplated a few solutions, and
decided to give this one a go.

This approach forces derive data to use Background SessionClass which has an
effect of not writing data to blobstore sync queue if the write to blobstore
was successful (it still writes data to the queue otherwise). This should
reduce the number of entries we write to the blobstore sync queue
significantly. The downside is that writes might get a bit slower - our
assumption is that this slowdown is acceptable. If that's not the case we can
always disable this option.

This diff overrides SessionClass for normal ::derive() method. However there's
also batch_derive() - this one will be addressed in the next diff.

One thing to note - we still write derived data mapping to blobstore sync queue. That should be find as we have a constant number of writes per commits.

Reviewed By: krallin

Differential Revision: D25910464

fbshipit-source-id: 4113d00bc0efe560fd14a5d4319b743d0a100dfa
This commit is contained in:
Stanislau Hlebik 2021-01-14 13:00:59 -08:00 committed by Facebook GitHub Bot
parent d71e5178f7
commit fca761e153
13 changed files with 39 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
eden/mononoke/derived_data/blame/derived.rs
View File

@ -57,7 +57,7 @@ impl BonsaiDerivable for BlameRoot {
type Options = BlameDeriveOptions;
async fn derive_from_parents(
async fn derive_from_parents_impl(
ctx: CoreContext,
repo: BlobRepo,
bonsai: BonsaiChangeset,

2
eden/mononoke/derived_data/changeset_info/derive.rs
View File

@ -28,7 +28,7 @@ impl BonsaiDerivable for ChangesetInfo {
type Options = ();
async fn derive_from_parents(
async fn derive_from_parents_impl(
_ctx: CoreContext,
_repo: BlobRepo,
bonsai: BonsaiChangeset,

2
eden/mononoke/derived_data/deleted_files_manifest/mapping.rs
View File

@ -54,7 +54,7 @@ impl BonsaiDerivable for RootDeletedManifestId {
type Options = ();
async fn derive_from_parents(
async fn derive_from_parents_impl(
ctx: CoreContext,
repo: BlobRepo,
bonsai: BonsaiChangeset,

2
eden/mononoke/derived_data/fastlog/mapping.rs
View File

@ -59,7 +59,7 @@ impl BonsaiDerivable for RootFastlog {
type Options = ();
async fn derive_from_parents(
async fn derive_from_parents_impl(
ctx: CoreContext,
repo: BlobRepo,
bonsai: BonsaiChangeset,

2
eden/mononoke/derived_data/filenodes/lib.rs
View File

@ -110,7 +110,7 @@ impl BonsaiDerivable for FilenodesOnlyPublic {
type Options = ();
async fn derive_from_parents(
async fn derive_from_parents_impl(
ctx: CoreContext,
repo: BlobRepo,
bonsai: BonsaiChangeset,

2
eden/mononoke/derived_data/fsnodes/mapping.rs
View File

@ -67,7 +67,7 @@ impl BonsaiDerivable for RootFsnodeId {
type Options = ();
async fn derive_from_parents(
async fn derive_from_parents_impl(
ctx: CoreContext,
repo: BlobRepo,
bonsai: BonsaiChangeset,

2
eden/mononoke/derived_data/mercurial_derived_data/mapping.rs
View File

@ -28,7 +28,7 @@ impl BonsaiDerivable for MappedHgChangesetId {
type Options = ();
async fn derive_from_parents(
async fn derive_from_parents_impl(
ctx: CoreContext,
repo: BlobRepo,
bonsai: BonsaiChangeset,

2
eden/mononoke/derived_data/skeleton_manifest/mapping.rs
View File

@ -69,7 +69,7 @@ impl BonsaiDerivable for RootSkeletonManifestId {
type Options = ();
async fn derive_from_parents(
async fn derive_from_parents_impl(
ctx: CoreContext,
repo: BlobRepo,
bonsai: BonsaiChangeset,

2
eden/mononoke/derived_data/src/derive_impl.rs
View File

@ -590,7 +590,7 @@ mod test {
type Options = ();
async fn derive_from_parents(
async fn derive_from_parents_impl(
_ctx: CoreContext,
_repo: BlobRepo,
bonsai: BonsaiChangeset,

27
eden/mononoke/derived_data/src/lib.rs
View File

@ -86,7 +86,7 @@ use anyhow::Error;
use async_trait::async_trait;
use auto_impl::auto_impl;
use blobrepo::BlobRepo;
use context::CoreContext;
use context::{CoreContext, SessionClass};
use lock_ext::LockExt;
use mononoke_types::{BonsaiChangeset, ChangesetId, RepositoryId};
use std::{
@ -124,15 +124,26 @@ pub trait BonsaiDerivable: Sized + 'static + Send + Sync + Clone {
/// Type for additional options to derivation
type Options: Send + Sync + 'static;
async fn derive_from_parents(
ctx: CoreContext,
repo: BlobRepo,
bonsai: BonsaiChangeset,
parents: Vec<Self>,
options: &Self::Options,
) -> Result<Self, Error> {
let ctx = override_ctx(ctx);
Self::derive_from_parents_impl(ctx, repo, bonsai, parents, options).await
}
/// Defines how to derive new representation for bonsai having derivations
/// for parents and having a current bonsai object.
///
/// Note that if any data has to be persistently stored in blobstore, mysql or any other store
/// then it's responsiblity of implementor of `derive_from_parents()` to save it.
/// then it's responsiblity of implementor of `derive_from_parents_impl()` to save it.
/// For example, to derive HgChangesetId we also need to derive all filenodes and all manifests
/// and then store them in blobstore. Derived data library is only responsible for
/// updating BonsaiDerivedMapping.
async fn derive_from_parents(
async fn derive_from_parents_impl(
ctx: CoreContext,
repo: BlobRepo,
bonsai: BonsaiChangeset,
@ -255,6 +266,16 @@ pub trait BonsaiDerived: Sized + 'static + Send + Sync + Clone + BonsaiDerivable
}
}
fn override_ctx(mut ctx: CoreContext) -> CoreContext {
if tunables::tunables().get_derived_data_use_background_session_class() {
ctx.session_mut()
.override_session_class(SessionClass::Background);
ctx
} else {
ctx
}
}
/// After derived data was generated then it will be stored in BonsaiDerivedMapping, which is
/// normally a persistent store. This is used to avoid regenerating the same derived data over
/// and over again.

2
eden/mononoke/derived_data/unodes/mapping.rs
View File

@ -60,7 +60,7 @@ impl BonsaiDerivable for RootUnodeManifestId {
type Options = UnodeVersion;
async fn derive_from_parents(
async fn derive_from_parents_impl(
ctx: CoreContext,
repo: BlobRepo,
bonsai: BonsaiChangeset,

2
eden/mononoke/git/git_types/src/derive_tree.rs
View File

@ -94,7 +94,7 @@ impl BonsaiDerivable for TreeHandle {
type Options = ();
async fn derive_from_parents(
async fn derive_from_parents_impl(
ctx: CoreContext,
repo: BlobRepo,
bonsai: BonsaiChangeset,

4
eden/mononoke/tunables/src/lib.rs
View File

@ -117,6 +117,10 @@ pub struct MononokeTunables {
backfill_write_qps: AtomicI64,
disable_commit_scribe_logging_scs: AtomicBool,
xrepo_sync_disable_all_syncs: AtomicBool,
// Use Background session class while deriving data. This makes derived data not write
// data to blobstore sync queue if a write was successful to the main blobstore.
derived_data_use_background_session_class: AtomicBool,
}
fn log_tunables(tunables: &TunablesStruct) -> String {