2020-10-30 03:35:41 +03:00
|
|
|
/*
|
|
|
|
* Copyright (c) Facebook, Inc. and its affiliates.
|
|
|
|
*
|
|
|
|
* This software may be used and distributed according to the terms of the
|
|
|
|
* GNU General Public License version 2.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#![deny(warnings)]
|
|
|
|
|
|
|
|
use std::sync::Arc;
|
|
|
|
use std::time::Duration;
|
|
|
|
|
2020-12-02 04:26:48 +03:00
|
|
|
use anyhow::{format_err, Context, Error};
|
2021-06-09 15:14:14 +03:00
|
|
|
use blobrepo::BlobRepo;
|
2020-12-02 18:25:35 +03:00
|
|
|
use clap::Arg;
|
2020-12-02 04:26:48 +03:00
|
|
|
use futures::future::join_all;
|
|
|
|
use slog::{error, info};
|
2020-10-30 03:35:41 +03:00
|
|
|
|
|
|
|
use blobstore_factory::{make_metadata_sql_factory, ReadOnlyStorage};
|
2021-03-25 17:32:21 +03:00
|
|
|
use bookmarks::{BookmarkName, Bookmarks};
|
2020-12-02 18:25:35 +03:00
|
|
|
use cmdlib::{
|
|
|
|
args::{self, MononokeMatches},
|
|
|
|
helpers,
|
|
|
|
};
|
2021-05-06 21:38:54 +03:00
|
|
|
use context::{CoreContext, SessionContainer};
|
2020-10-30 03:35:41 +03:00
|
|
|
use fbinit::FacebookInit;
|
|
|
|
use metaconfig_types::MetadataDatabaseConfig;
|
2021-03-24 03:06:19 +03:00
|
|
|
use segmented_changelog::{SegmentedChangelogSqlConnections, SegmentedChangelogTailer};
|
2020-10-30 03:35:41 +03:00
|
|
|
use sql_ext::facebook::MyAdmin;
|
|
|
|
use sql_ext::replication::{NoReplicaLagMonitor, ReplicaLagMonitor};
|
|
|
|
|
|
|
|
const ONCE_ARG: &str = "once";
|
2020-12-02 04:26:48 +03:00
|
|
|
const REPO_ARG: &str = "repo";
|
2020-10-30 03:35:41 +03:00
|
|
|
|
|
|
|
#[fbinit::main]
|
|
|
|
fn main(fb: FacebookInit) -> Result<(), Error> {
|
2020-12-01 22:43:16 +03:00
|
|
|
let app = args::MononokeAppBuilder::new("Updates segmented changelog assets.")
|
2021-05-07 12:06:05 +03:00
|
|
|
.with_scuba_logging_args()
|
2020-10-30 03:35:41 +03:00
|
|
|
.with_advanced_args_hidden()
|
|
|
|
.with_fb303_args()
|
|
|
|
.build()
|
|
|
|
.about("Builds a new version of segmented changelog.")
|
|
|
|
.arg(
|
2020-12-02 04:26:48 +03:00
|
|
|
Arg::with_name(REPO_ARG)
|
|
|
|
.long(REPO_ARG)
|
|
|
|
.takes_value(true)
|
|
|
|
.required(true)
|
|
|
|
.multiple(true)
|
|
|
|
.help("Repository name to warm-up"),
|
|
|
|
)
|
2020-10-30 03:35:41 +03:00
|
|
|
.arg(
|
|
|
|
Arg::with_name(ONCE_ARG)
|
|
|
|
.long(ONCE_ARG)
|
|
|
|
.takes_value(false)
|
|
|
|
.required(false)
|
|
|
|
.help("When set, the tailer will perform a single incremental build run."),
|
|
|
|
);
|
2021-04-16 20:26:03 +03:00
|
|
|
let matches = app.get_matches(fb)?;
|
2020-10-30 03:35:41 +03:00
|
|
|
|
2021-04-16 20:26:03 +03:00
|
|
|
let logger = matches.logger();
|
2021-05-06 21:38:54 +03:00
|
|
|
let session = SessionContainer::new_with_defaults(fb);
|
|
|
|
let ctx = session.new_context(logger.clone(), matches.scuba_sample_builder());
|
2020-10-30 03:35:41 +03:00
|
|
|
helpers::block_execute(
|
|
|
|
run(ctx, &matches),
|
|
|
|
fb,
|
|
|
|
&std::env::var("TW_JOB_NAME").unwrap_or_else(|_| "segmented_changelog_tailer".to_string()),
|
2021-04-16 20:26:03 +03:00
|
|
|
logger,
|
2020-10-30 03:35:41 +03:00
|
|
|
&matches,
|
|
|
|
cmdlib::monitoring::AliveService,
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
2020-12-02 18:25:35 +03:00
|
|
|
async fn run<'a>(ctx: CoreContext, matches: &'a MononokeMatches<'a>) -> Result<(), Error> {
|
2020-12-02 04:26:48 +03:00
|
|
|
let reponames: Vec<_> = matches
|
|
|
|
.values_of(REPO_ARG)
|
|
|
|
.ok_or_else(|| format_err!("--{} argument is required", REPO_ARG))?
|
|
|
|
.map(ToString::to_string)
|
|
|
|
.collect();
|
|
|
|
if reponames.is_empty() {
|
|
|
|
error!(ctx.logger(), "At least one repo had to be specified");
|
|
|
|
return Ok(());
|
|
|
|
}
|
2020-10-30 03:35:41 +03:00
|
|
|
|
2021-04-16 20:26:03 +03:00
|
|
|
let config_store = matches.config_store();
|
mononoke: use MononokeEnvironment in RepoFactory
Summary:
There is a very frustrating operation that happens often when working on the
Mononoke code base:
- You want to add a flag
- You want to consume it in the repo somewhere
Unfortunately, when we need to do this, we end up having to thread this from a
million places and parse it out in every single main() we have.
This is a mess, and it results in every single Mononoke binary starting with
heaps of useless boilerplate:
```
let matches = app.get_matches();
let (caching, logger, mut runtime) = matches.init_mononoke(fb)?;
let config_store = args::init_config_store(fb, &logger, &matches)?;
let mysql_options = args::parse_mysql_options(&matches);
let blobstore_options = args::parse_blobstore_options(&matches)?;
let readonly_storage = args::parse_readonly_storage(&matches);
```
So, this diff updates us to just use MononokeEnvironment directly in
RepoFactory, which means none of that has to happen: we can now add a flag,
parse it into MononokeEnvironment, and get going.
While we're at it, we can also remove blobstore options and all that jazz from
MononokeApiEnvironment since now it's there in the underlying RepoFactory.
Reviewed By: HarveyHunt
Differential Revision: D27767700
fbshipit-source-id: e1e359bf403b4d3d7b36e5f670aa1a7dd4f1d209
2021-04-16 20:26:03 +03:00
|
|
|
let mysql_options = matches.mysql_options();
|
2020-12-02 04:26:48 +03:00
|
|
|
let configs = args::load_repo_configs(config_store, matches)?;
|
2020-10-30 03:35:41 +03:00
|
|
|
let readonly_storage = ReadOnlyStorage(false);
|
|
|
|
|
2020-12-02 04:26:48 +03:00
|
|
|
let mut tasks = Vec::new();
|
|
|
|
for (index, reponame) in reponames.into_iter().enumerate() {
|
|
|
|
let config = configs
|
|
|
|
.repos
|
|
|
|
.get(&reponame)
|
|
|
|
.ok_or_else(|| format_err!("unknown repository: {}", reponame))?;
|
|
|
|
let repo_id = config.repoid;
|
2021-02-25 03:49:54 +03:00
|
|
|
|
|
|
|
let bookmark_name = &config.segmented_changelog_config.master_bookmark;
|
|
|
|
let track_bookmark = BookmarkName::new(bookmark_name).with_context(|| {
|
|
|
|
format!(
|
|
|
|
"error parsing the name of the bookmark to track: {}",
|
|
|
|
bookmark_name,
|
|
|
|
)
|
|
|
|
})?;
|
|
|
|
|
2020-12-02 04:26:48 +03:00
|
|
|
info!(
|
|
|
|
ctx.logger(),
|
|
|
|
"repo name '{}' translates to id {}", reponame, repo_id
|
|
|
|
);
|
|
|
|
|
|
|
|
let storage_config = config.storage_config.clone();
|
|
|
|
let db_address = match &storage_config.metadata {
|
|
|
|
MetadataDatabaseConfig::Local(_) => None,
|
|
|
|
MetadataDatabaseConfig::Remote(remote_config) => {
|
|
|
|
Some(remote_config.primary.db_address.clone())
|
|
|
|
}
|
|
|
|
};
|
|
|
|
let replica_lag_monitor: Arc<dyn ReplicaLagMonitor> = match db_address {
|
|
|
|
None => Arc::new(NoReplicaLagMonitor()),
|
|
|
|
Some(address) => {
|
|
|
|
let my_admin = MyAdmin::new(ctx.fb).context("building myadmin client")?;
|
|
|
|
Arc::new(my_admin.single_shard_lag_monitor(address))
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
let sql_factory = make_metadata_sql_factory(
|
|
|
|
ctx.fb,
|
|
|
|
storage_config.metadata,
|
2020-12-18 02:44:56 +03:00
|
|
|
mysql_options.clone(),
|
2020-12-02 04:26:48 +03:00
|
|
|
readonly_storage,
|
|
|
|
)
|
2020-10-30 03:35:41 +03:00
|
|
|
.await
|
2020-12-02 04:26:48 +03:00
|
|
|
.with_context(|| format!("repo {}: constructing metadata sql factory", repo_id))?;
|
|
|
|
|
2021-03-18 06:10:36 +03:00
|
|
|
let segmented_changelog_sql_connections = sql_factory
|
|
|
|
.open::<SegmentedChangelogSqlConnections>()
|
2020-12-02 04:26:48 +03:00
|
|
|
.with_context(|| {
|
2021-03-18 06:10:36 +03:00
|
|
|
format!(
|
|
|
|
"repo {}: error constructing segmented changelog sql connections",
|
|
|
|
repo_id
|
|
|
|
)
|
2020-12-02 04:26:48 +03:00
|
|
|
})?;
|
|
|
|
|
|
|
|
// This is a bit weird from the dependency point of view but I think that it is best. The
|
|
|
|
// BlobRepo may have a SegmentedChangelog attached to it but that doesn't hurt us in any
|
|
|
|
// way. On the other hand reconstructing the dependencies for SegmentedChangelog without
|
|
|
|
// BlobRepo is probably prone to more problems from the maintenance perspective.
|
2021-06-09 15:14:14 +03:00
|
|
|
let blobrepo: BlobRepo =
|
|
|
|
args::open_repo_with_repo_id(ctx.fb, ctx.logger(), repo_id, matches).await?;
|
2021-03-24 03:06:19 +03:00
|
|
|
let segmented_changelog_tailer = SegmentedChangelogTailer::new(
|
|
|
|
repo_id,
|
|
|
|
segmented_changelog_sql_connections,
|
|
|
|
replica_lag_monitor,
|
|
|
|
blobrepo.get_changeset_fetcher(),
|
|
|
|
Arc::new(blobrepo.get_blobstore()),
|
2021-03-25 17:32:21 +03:00
|
|
|
Arc::clone(blobrepo.bookmarks()) as Arc<dyn Bookmarks>,
|
2021-03-24 03:06:19 +03:00
|
|
|
track_bookmark,
|
2021-05-07 22:22:58 +03:00
|
|
|
None,
|
2021-03-24 03:06:19 +03:00
|
|
|
);
|
2020-12-02 04:26:48 +03:00
|
|
|
|
|
|
|
info!(
|
|
|
|
ctx.logger(),
|
|
|
|
"repo {}: SegmentedChangelogTailer initialized", repo_id
|
|
|
|
);
|
|
|
|
|
|
|
|
if matches.is_present(ONCE_ARG) {
|
|
|
|
segmented_changelog_tailer
|
|
|
|
.once(&ctx)
|
|
|
|
.await
|
|
|
|
.with_context(|| format!("repo {}: incrementally building repo", repo_id))?;
|
|
|
|
info!(
|
|
|
|
ctx.logger(),
|
|
|
|
"repo {}: SegmentedChangelogTailer is done", repo_id,
|
|
|
|
);
|
2021-02-25 03:49:54 +03:00
|
|
|
} else if let Some(period) = config.segmented_changelog_config.tailer_update_period {
|
|
|
|
// spread out update operations, start updates on another repo after 7 seconds
|
|
|
|
let wait_to_start = Duration::from_secs(7 * index as u64);
|
2020-12-02 04:26:48 +03:00
|
|
|
let ctx = ctx.clone();
|
|
|
|
tasks.push(async move {
|
mononoke: update to tokio 1.x
Summary:
NOTE: there is one final pre-requisite here, which is that we should default all Mononoke binaries to `--use-mysql-client` because the other SQL client implementations will break once this lands. That said, this is probably the right time to start reviewing.
There's a lot going on here, but Tokio updates being what they are, it has to happen as just one diff (though I did try to minimize churn by modernizing a bunch of stuff in earlier diffs).
Here's a detailed list of what is going on:
- I had to add a number `cargo_toml_dir` for binaries in `eden/mononoke/TARGETS`, because we have to use 2 versions of Bytes concurrently at this time, and the two cannot co-exist in the same Cargo workspace.
- Lots of little Tokio changes:
- Stream abstractions moving to `tokio-stream`
- `tokio::time::delay_for` became `tokio::time::sleep`
- `tokio::sync::watch::Sender::send` became `tokio::sync::watch::Sender::broadcast`
- `tokio::sync::Semaphore::acquire` returns a `Result` now.
- `tokio::runtime::Runtime::block_on` no longer takes a `&mut self` (just a `&self`).
- `Notify` grew a few more methods with different semantics. We only use this in tests, I used what seemed logical given the use case.
- Runtime builders have changed quite a bit:
- My `no_coop` patch is gone in Tokio 1.x, but it has a new `tokio::task::unconstrained` wrapper (also from me), which I included on `MononokeApi::new`.
- Tokio now detects your logical CPUs, not physical CPUs, so we no longer need to use `num_cpus::get()` to figure it out.
- Tokio 1.x now uses Bytes 1.x:
- At the edges (i.e. streams returned to Hyper or emitted by RepoClient), we need to return Bytes 1.x. However, internally we still use Bytes 0.5 in some places (notably: Filestore).
- In LFS, this means we make a copy. We used to do that a while ago anyway (in the other direction) and it was never a meaningful CPU cost, so I think this is fine.
- In Mononoke Server it doesn't really matter because that still generates ... Bytes 0.1 anyway so there was a copy before from 0.1 to 0.5 and it's from 0.1 to 1.x.
- In the very few places where we read stuff using Tokio from the outside world (historical import tools for LFS), we copy.
- tokio-tls changed a lot, they removed all the convenience methods around connecting. This resulted in updates to:
- How we listen in Mononoke Server & LFS
- How we connect in hgcli.
- Note: all this stuff has test coverage.
- The child process API changed a little bit. We used to have a ChildWrapper around the hg sync job to make a Tokio 0.2.x child look more like a Tokio 1.x Child, so now we can just remove this.
- Hyper changed their Websocket upgrade mechanism (you now need the whole `Request` to upgrade, whereas before that you needed just the `Body`, so I changed up our code a little bit in Mononoke's HTTP acceptor to defer splitting up the `Request` into parts until after we know whether we plan to upgrade it.
- I removed the MySQL tests that didn't use mysql client, because we're leaving that behind and don't intend to support it on Tokio 1.x.
Reviewed By: mitrandir77
Differential Revision: D26669620
fbshipit-source-id: acb6aff92e7f70a7a43f32cf758f252f330e60c9
2021-04-28 17:35:21 +03:00
|
|
|
tokio::time::sleep(wait_to_start).await;
|
2021-02-25 03:49:54 +03:00
|
|
|
segmented_changelog_tailer.run(&ctx, period).await;
|
2020-12-02 04:26:48 +03:00
|
|
|
});
|
|
|
|
}
|
2020-10-30 03:35:41 +03:00
|
|
|
}
|
|
|
|
|
2020-12-02 04:26:48 +03:00
|
|
|
join_all(tasks).await;
|
|
|
|
|
2020-10-30 03:35:41 +03:00
|
|
|
Ok(())
|
|
|
|
}
|