mirror of
https://github.com/facebook/sapling.git
synced 2024-10-09 08:18:15 +03:00
mononoke/blobstore_healer: make it OSS buildable
Reviewed By: farnz Differential Revision: D22460549 fbshipit-source-id: aa5327f5dae1008cee784d41e322034cd0bb5b61
This commit is contained in:
Lukas Piatkowski
Facebook GitHub Bot
parent
6b9637bbac
commit
a41db27baf
@ -4,7 +4,7 @@ edition = "2018"
|
||||
version = "0.1.0"
|
||||
authors = ['Facebook']
|
||||
license = "GPLv2+"
|
||||
include = ["cmds/admin/**/*.rs", "cmds/aliasverify.rs", "cmds/backfill_derived_data/**/*.rs", "cmds/backfill_git_mapping.rs", "cmds/benchmark_filestore.rs", "cmds/benchmark_storage_config/**/*.rs", "cmds/blobimport.rs", "cmds/bonsai_verify/**/*.rs", "cmds/configlint.rs", "cmds/dumprev.rs", "cmds/idxdump.rs", "cmds/lfs_import.rs", "cmds/manual_scrub/**/*.rs", "cmds/rechunker.rs", "cmds/revlogrepo.rs", "cmds/statistics_collector.rs", "cmds/upload_globalrevs.rs"]
|
||||
include = ["cmds/admin/**/*.rs", "cmds/aliasverify.rs", "cmds/backfill_derived_data/**/*.rs", "cmds/backfill_git_mapping.rs", "cmds/benchmark_filestore.rs", "cmds/benchmark_storage_config/**/*.rs", "cmds/blobimport.rs", "cmds/blobstore_healer/**/*.rs", "cmds/bonsai_verify/**/*.rs", "cmds/configlint.rs", "cmds/dumprev.rs", "cmds/idxdump.rs", "cmds/lfs_import.rs", "cmds/manual_scrub/**/*.rs", "cmds/rechunker.rs", "cmds/revlogrepo.rs", "cmds/statistics_collector.rs", "cmds/upload_globalrevs.rs"]
|
||||
|
||||
[[bin]]
|
||||
name = "admin"
|
||||
@ -34,6 +34,10 @@ path = "cmds/benchmark_storage_config/main.rs"
|
||||
name = "blobimport"
|
||||
path = "cmds/blobimport.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "blobstore_healer"
|
||||
path = "cmds/blobstore_healer/main.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "bonsai_verify"
|
||||
path = "cmds/bonsai_verify/main.rs"
|
||||
@ -84,6 +88,7 @@ blobrepo_override = { path = "blobrepo/override" }
|
||||
blobrepo_utils = { path = "blobrepo_utils" }
|
||||
blobstore = { path = "blobstore" }
|
||||
blobstore_factory = { path = "blobstore/factory" }
|
||||
blobstore_sync_queue = { path = "blobstore_sync_queue" }
|
||||
bonsai_globalrev_mapping = { path = "bonsai_globalrev_mapping" }
|
||||
bookmark_renaming = { path = "commit_rewriting/bookmark_renaming" }
|
||||
bookmarks = { path = "bookmarks" }
|
||||
@ -120,12 +125,14 @@ redactedblobstore = { path = "blobstore/redactedblobstore" }
|
||||
revset = { path = "revset" }
|
||||
scuba_ext = { path = "common/scuba_ext" }
|
||||
skiplist = { path = "reachabilityindex/skiplist" }
|
||||
sql_construct = { path = "common/sql_construct" }
|
||||
sql_ext = { path = "common/rust/sql_ext" }
|
||||
sqlblob = { path = "blobstore/sqlblob" }
|
||||
synced_commit_mapping = { path = "commit_rewriting/synced_commit_mapping" }
|
||||
throttledblob = { path = "blobstore/throttledblob" }
|
||||
unodes = { path = "derived_data/unodes" }
|
||||
xdiff = { path = "../scm/lib/xdiff" }
|
||||
cached_config = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
||||
cachelib = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
||||
cloned = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
||||
failure_ext = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
||||
@ -134,6 +141,7 @@ fbthrift = { git = "https://github.com/facebook/fbthrift.git", branch = "master"
|
||||
futures_ext = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
||||
futures_stats = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
||||
lock_ext = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
||||
sql = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
||||
stats = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
||||
time_ext = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
||||
tracing = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
||||
@ -141,12 +149,14 @@ anyhow = "1.0"
|
||||
ascii = "1.0"
|
||||
async-trait = "0.1.29"
|
||||
bytes = { version = "0.5", features = ["serde"] }
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
clap = "2.33"
|
||||
criterion = "0.3"
|
||||
futures = { version = "0.3.5", features = ["async-await", "compat"] }
|
||||
futures-old = { package = "futures", version = "0.1" }
|
||||
futures-util = "0.3"
|
||||
itertools = "0.8"
|
||||
lazy_static = "1.0"
|
||||
rand = { version = "0.7", features = ["small_rng"] }
|
||||
serde = { version = "1.0", features = ["derive", "rc"] }
|
||||
serde_derive = "1.0"
|
||||
@ -163,7 +173,6 @@ toml = "=0.5.6"
|
||||
|
||||
[dev-dependencies]
|
||||
fixtures = { path = "tests/fixtures" }
|
||||
sql_construct = { path = "common/sql_construct" }
|
||||
tests_utils = { path = "tests/utils" }
|
||||
maplit = "1.0"
|
||||
|
||||
|
||||
@ -9,13 +9,14 @@
|
||||
#![feature(never_type)]
|
||||
|
||||
mod dummy;
|
||||
#[cfg(fbcode_build)]
|
||||
mod facebook;
|
||||
mod healer;
|
||||
|
||||
use anyhow::{bail, format_err, Context, Error, Result};
|
||||
use blobstore::Blobstore;
|
||||
use blobstore_factory::{make_blobstore, BlobstoreOptions, ReadOnlyStorage};
|
||||
use blobstore_sync_queue::{BlobstoreSyncQueue, SqlBlobstoreSyncQueue};
|
||||
use cached_config::ConfigStore;
|
||||
use chrono::Duration as ChronoDuration;
|
||||
use clap::{value_t, App, Arg};
|
||||
use cmdlib::{
|
||||
@ -30,21 +31,18 @@ use healer::Healer;
|
||||
use lazy_static::lazy_static;
|
||||
use metaconfig_types::{BlobConfig, DatabaseConfig, LocalDatabaseConfig, StorageConfig};
|
||||
use mononoke_types::DateTime;
|
||||
use slog::{info, o, warn};
|
||||
use slog::{info, o};
|
||||
use sql::Connection;
|
||||
use sql_construct::SqlConstructFromDatabaseConfig;
|
||||
use sql_ext::facebook::myrouter_ready;
|
||||
use sql_ext::{
|
||||
facebook::MysqlOptions,
|
||||
facebook::{myrouter_ready, MysqlOptions},
|
||||
open_sqlite_path,
|
||||
replication::{LaggableCollectionMonitor, ReplicaLagMonitor, WaitForReplicationConfig},
|
||||
};
|
||||
use sql_facebook::{myrouter, raw};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
const CONFIGERATOR_REGIONS_CONFIG: &str = "myrouter/regions.json";
|
||||
const QUIET_ARG: &'static str = "quiet";
|
||||
const ITER_LIMIT_ARG: &'static str = "iteration-limit";
|
||||
const HEAL_MIN_AGE_ARG: &'static str = "heal-min-age-secs";
|
||||
@ -55,50 +53,6 @@ lazy_static! {
|
||||
static ref DEFAULT_ENTRY_HEALING_MIN_AGE: ChronoDuration = ChronoDuration::minutes(2);
|
||||
}
|
||||
|
||||
async fn open_mysql_raw_replicas(
|
||||
fb: FacebookInit,
|
||||
ctx: &CoreContext,
|
||||
tier: &str,
|
||||
regions: Arc<Vec<String>>,
|
||||
) -> Result<Vec<(String, Connection)>, Error> {
|
||||
let raw_conns = regions.iter().cloned().map({
|
||||
move |region| async move {
|
||||
let mut conn_builder = raw::Builder::new(tier, raw::InstanceRequirement::ReplicaOnly);
|
||||
conn_builder.role_override("scriptro");
|
||||
conn_builder.explicit_region(®ion);
|
||||
|
||||
let conn = match conn_builder.build(fb).compat().await {
|
||||
Ok(c) =>
|
||||
Some(Connection::Mysql(c)),
|
||||
Err(_e) => {
|
||||
warn!(ctx.logger(),
|
||||
"Could not connect to a replica in {}, likely that region does not have one.", region);
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
(region, conn)
|
||||
}
|
||||
});
|
||||
|
||||
let filtered: Vec<_> = future::join_all(raw_conns)
|
||||
.await
|
||||
.into_iter()
|
||||
.filter_map(|(region, conn)| match conn {
|
||||
Some(conn) => Some((region, conn)),
|
||||
None => None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
info!(
|
||||
ctx.logger(),
|
||||
"Monitoring regions: {:?}",
|
||||
filtered.iter().map(|(r, _)| r).collect::<Vec<_>>()
|
||||
);
|
||||
|
||||
Ok(filtered)
|
||||
}
|
||||
|
||||
async fn maybe_schedule_healer_for_storage(
|
||||
fb: FacebookInit,
|
||||
ctx: &CoreContext,
|
||||
@ -182,29 +136,14 @@ async fn maybe_schedule_healer_for_storage(
|
||||
vec![("sqlite_region".to_string(), Connection::with_sqlite(c))]
|
||||
}
|
||||
DatabaseConfig::Remote(remote) => {
|
||||
let db_address = remote.db_address;
|
||||
let regions = ConfigStore::configerator(fb, None, None, Duration::from_secs(5))?
|
||||
.get_config_handle::<Vec<String>>(CONFIGERATOR_REGIONS_CONFIG.to_owned())?
|
||||
.get();
|
||||
info!(ctx.logger(), "Discovered regions: {:?}", *regions);
|
||||
if let Some(myrouter_port) = mysql_options.myrouter_port {
|
||||
let mut conn_builder = myrouter::Builder::new();
|
||||
conn_builder
|
||||
.service_type(myrouter::ServiceType::SLAVE)
|
||||
.locality(myrouter::DbLocality::EXPLICIT)
|
||||
.tier(db_address, None)
|
||||
.port(myrouter_port);
|
||||
|
||||
regions
|
||||
.iter()
|
||||
.map(|region| {
|
||||
conn_builder.explicit_region(region.clone());
|
||||
let conn: Connection = conn_builder.build_read_only().into();
|
||||
(region.clone(), conn)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
} else {
|
||||
open_mysql_raw_replicas(fb, ctx, &db_address, regions).await?
|
||||
#[cfg(fbcode_build)]
|
||||
{
|
||||
facebook::open_remote_db(fb, &ctx, remote, mysql_options).await?
|
||||
}
|
||||
#[cfg(not(fbcode_build))]
|
||||
{
|
||||
let _ = remote;
|
||||
unimplemented!("Remote DB is not yet implemented for non fbcode builds");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
Loading…
Reference in New Issue
Block a user