2020-03-04 17:06:03 +03:00
|
|
|
/*
|
|
|
|
* Copyright (c) Facebook, Inc. and its affiliates.
|
|
|
|
*
|
|
|
|
* This software may be used and distributed according to the terms of the
|
|
|
|
* GNU General Public License version 2.
|
|
|
|
*/
|
|
|
|
|
|
|
|
use anyhow::{anyhow, Error};
|
2020-12-02 18:25:35 +03:00
|
|
|
use args::MononokeClapApp;
|
2020-03-04 17:06:03 +03:00
|
|
|
use ascii::AsciiStr;
|
|
|
|
use blobrepo::BlobRepo;
|
2020-06-22 17:20:47 +03:00
|
|
|
use blobrepo_hg::BlobRepoHg;
|
2020-03-04 17:06:03 +03:00
|
|
|
use blobstore::Loadable;
|
2021-02-17 23:40:17 +03:00
|
|
|
use clap::{Arg, ArgGroup};
|
2020-03-04 17:06:03 +03:00
|
|
|
use cloned::cloned;
|
|
|
|
use cmdlib::args;
|
|
|
|
use context::CoreContext;
|
|
|
|
use fbinit::FacebookInit;
|
2020-12-17 18:43:34 +03:00
|
|
|
use futures::stream;
|
2020-03-04 17:06:03 +03:00
|
|
|
use futures_util::future::TryFutureExt;
|
|
|
|
use futures_util::stream::{StreamExt, TryStreamExt};
|
|
|
|
use mercurial_types::HgChangesetId;
|
|
|
|
use std::fs;
|
|
|
|
use std::io::{self, BufRead};
|
|
|
|
use std::path::Path;
|
|
|
|
|
2020-12-02 18:25:35 +03:00
|
|
|
fn setup_app<'a, 'b>() -> MononokeClapApp<'a, 'b> {
|
2020-12-01 22:43:16 +03:00
|
|
|
args::MononokeAppBuilder::new("Tool to backfill git mappings for given commits")
|
2020-03-04 17:06:03 +03:00
|
|
|
.build()
|
2021-02-17 23:40:17 +03:00
|
|
|
.arg(Arg::with_name("git").long("git"))
|
|
|
|
.arg(Arg::with_name("svnrev").long("svnrev"))
|
|
|
|
.group(
|
|
|
|
ArgGroup::with_name("mode")
|
|
|
|
.args(&["git", "svnrev"])
|
|
|
|
.required(true),
|
|
|
|
)
|
2020-03-04 17:06:03 +03:00
|
|
|
.arg(Arg::from_usage(
|
|
|
|
"<IN_FILENAME> 'file with hg changeset ids (separated by newlines)'",
|
|
|
|
))
|
|
|
|
}
|
|
|
|
|
|
|
|
fn parse_input<P: AsRef<Path>>(
|
|
|
|
file: P,
|
|
|
|
) -> Result<impl Iterator<Item = Result<HgChangesetId, Error>>, Error> {
|
|
|
|
let file = fs::File::open(file)?;
|
|
|
|
let iter = io::BufReader::new(file)
|
|
|
|
.lines()
|
|
|
|
.map(|line| HgChangesetId::from_ascii_str(AsciiStr::from_ascii(&line?)?));
|
|
|
|
Ok(iter)
|
|
|
|
}
|
|
|
|
|
2021-02-17 23:40:17 +03:00
|
|
|
#[derive(Debug, Copy, Clone)]
|
|
|
|
pub enum BackfillMode {
|
|
|
|
Git,
|
|
|
|
Svnrev,
|
|
|
|
}
|
|
|
|
|
2020-03-04 17:06:03 +03:00
|
|
|
pub async fn backfill<P: AsRef<Path>>(
|
|
|
|
ctx: CoreContext,
|
|
|
|
repo: BlobRepo,
|
|
|
|
in_path: P,
|
2021-02-17 23:40:17 +03:00
|
|
|
mode: BackfillMode,
|
2020-03-04 17:06:03 +03:00
|
|
|
) -> Result<(), Error> {
|
|
|
|
let chunk_size = 1000;
|
|
|
|
let ids = parse_input(in_path)?;
|
|
|
|
stream::iter(ids)
|
|
|
|
.and_then(|hg_cs_id| {
|
|
|
|
cloned!(ctx, repo);
|
|
|
|
async move {
|
2020-06-04 22:18:22 +03:00
|
|
|
let id = repo
|
|
|
|
.get_bonsai_from_hg(ctx.clone(), hg_cs_id)
|
|
|
|
.await?
|
|
|
|
.ok_or(anyhow!("hg commit {} is missing", hg_cs_id))?;
|
2020-03-04 17:06:03 +03:00
|
|
|
Ok(id)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
.map_ok({
|
|
|
|
cloned!(ctx, repo);
|
|
|
|
move |id| {
|
2020-11-20 16:50:00 +03:00
|
|
|
cloned!(ctx, repo);
|
2020-11-27 14:29:09 +03:00
|
|
|
async move { id.load(&ctx, repo.blobstore()).await }.map_err(anyhow::Error::from)
|
2020-03-04 17:06:03 +03:00
|
|
|
}
|
|
|
|
})
|
|
|
|
.try_buffer_unordered(chunk_size)
|
|
|
|
// TryStreamExt doesn't have the try_chunks method yet so we have to do it by folding.
|
|
|
|
.chunks(chunk_size)
|
|
|
|
.map(|chunk| chunk.into_iter().collect::<Result<Vec<_>, _>>())
|
|
|
|
.try_for_each(|chunk| {
|
|
|
|
cloned!(ctx, repo);
|
|
|
|
async move {
|
2021-02-17 23:40:17 +03:00
|
|
|
match mode {
|
|
|
|
BackfillMode::Git => {
|
|
|
|
repo.bonsai_git_mapping()
|
|
|
|
.bulk_import_from_bonsai(&ctx, &chunk)
|
|
|
|
.await
|
|
|
|
}
|
|
|
|
BackfillMode::Svnrev => {
|
|
|
|
repo.bonsai_svnrev_mapping()
|
|
|
|
.bulk_import_from_bonsai(&ctx, &chunk)
|
|
|
|
.await
|
|
|
|
}
|
|
|
|
}
|
2020-03-04 17:06:03 +03:00
|
|
|
}
|
|
|
|
})
|
|
|
|
.await
|
|
|
|
}
|
|
|
|
|
|
|
|
#[fbinit::main]
|
|
|
|
fn main(fb: FacebookInit) -> Result<(), Error> {
|
2021-04-16 20:26:03 +03:00
|
|
|
let matches = setup_app().get_matches(fb)?;
|
2020-03-04 17:06:03 +03:00
|
|
|
|
2021-04-16 20:26:03 +03:00
|
|
|
let logger = matches.logger();
|
|
|
|
let runtime = matches.runtime();
|
2020-03-04 17:06:03 +03:00
|
|
|
|
|
|
|
let ctx = CoreContext::new_with_logger(fb, logger.clone());
|
|
|
|
|
2021-02-17 23:40:17 +03:00
|
|
|
let mode = if matches.is_present("git") {
|
|
|
|
BackfillMode::Git
|
|
|
|
} else if matches.is_present("svnrev") {
|
|
|
|
BackfillMode::Svnrev
|
|
|
|
} else {
|
|
|
|
panic!("backfill mode not specified");
|
|
|
|
};
|
|
|
|
|
2020-03-04 17:06:03 +03:00
|
|
|
let run = async {
|
2020-10-30 21:53:19 +03:00
|
|
|
let repo = args::open_repo(fb, &logger, &matches).await?;
|
2020-03-04 17:06:03 +03:00
|
|
|
let in_filename = matches.value_of("IN_FILENAME").unwrap();
|
2021-02-17 23:40:17 +03:00
|
|
|
backfill(ctx, repo, in_filename, mode).await
|
2020-03-04 17:06:03 +03:00
|
|
|
};
|
|
|
|
|
2021-02-22 20:20:44 +03:00
|
|
|
runtime.block_on(run)?;
|
2020-03-04 17:06:03 +03:00
|
|
|
Ok(())
|
|
|
|
}
|