mirror of
https://github.com/facebook/sapling.git
synced 2024-10-09 08:18:15 +03:00
mononoke: make chunk size configurable in regenerate_filenodes binary
Summary: It's useful to have it configurable. While here, also use slog instead of println to attach timestamp as well Reviewed By: Croohand Differential Revision: D29165693 fbshipit-source-id: d844926560b15042445d5861a281870ac102d12e
This commit is contained in:
parent
d12a1fe872
commit
fbc07cb4c3
@ -18,6 +18,7 @@ derived_data_filenodes = { version = "0.1.0", path = "../derived_data/filenodes"
|
||||
fbinit = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
|
||||
futures = { version = "0.3.13", features = ["async-await", "compat"] }
|
||||
mercurial_types = { version = "0.1.0", path = "../mercurial/types" }
|
||||
slog = { version = "2.5", features = ["max_level_trace"] }
|
||||
|
||||
[patch.crates-io]
|
||||
addr2line = { git = "https://github.com/gimli-rs/addr2line.git", rev = "0b6b6018b5b252a18e628fba03885f7d21844b3c" }
|
||||
|
@ -21,6 +21,7 @@ use fbinit::FacebookInit;
|
||||
use futures::compat::Future01CompatExt;
|
||||
use futures::future::{join_all, FutureExt};
|
||||
use mercurial_types::{HgChangesetId, HgNodeHash};
|
||||
use slog::info;
|
||||
use std::fs::File;
|
||||
use std::str::FromStr;
|
||||
use std::{
|
||||
@ -66,9 +67,9 @@ async fn regenerate_all_manifests(
|
||||
ctx: CoreContext,
|
||||
repo: BlobRepo,
|
||||
hg_css: Vec<HgChangesetId>,
|
||||
chunk_size: usize,
|
||||
) -> Result<(), Error> {
|
||||
let mut i = 0;
|
||||
let chunk_size = 100;
|
||||
for chunk in hg_css.chunks(chunk_size) {
|
||||
let mut futs = vec![];
|
||||
for hg_cs in chunk {
|
||||
@ -77,11 +78,11 @@ async fn regenerate_all_manifests(
|
||||
repo.clone(),
|
||||
*hg_cs,
|
||||
));
|
||||
i += 1;
|
||||
}
|
||||
let res: Result<Vec<_>, Error> = join_all(futs).await.into_iter().collect();
|
||||
res?;
|
||||
i += chunk_size;
|
||||
println!("processed {}", i);
|
||||
info!(ctx.logger(), "processed {}", i);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@ -94,6 +95,7 @@ fn main(fb: FacebookInit) -> Result<(), Error> {
|
||||
r#"
|
||||
--i-know-what-i-am-doing
|
||||
--file [FILE] 'contains list of commit hashes for which we want to regenerate filenodes'
|
||||
--chunk-size [SIZE] 'how many commits to process at once (default: 100)'
|
||||
"#,
|
||||
)
|
||||
.get_matches(fb)?;
|
||||
@ -112,6 +114,8 @@ fn main(fb: FacebookInit) -> Result<(), Error> {
|
||||
|
||||
let inputfile = matches.value_of("file").expect("input file is not set");
|
||||
let inputfile = File::open(inputfile).expect("cannot open input file");
|
||||
|
||||
let chunk_size = args::get_usize(&matches, "chunk-size", 100);
|
||||
let file = BufReader::new(&inputfile);
|
||||
rt.block_on(
|
||||
regenerate_all_manifests(
|
||||
@ -120,6 +124,7 @@ fn main(fb: FacebookInit) -> Result<(), Error> {
|
||||
file.lines()
|
||||
.map(|line| convert_to_cs(&line.unwrap()).unwrap())
|
||||
.collect(),
|
||||
chunk_size,
|
||||
)
|
||||
.boxed(),
|
||||
)
|
||||
|
@ -113,7 +113,7 @@
|
||||
$ hg log -r 'all()' -T '{node}\n' > hashes
|
||||
$ regenerate_hg_filenodes --file 'hashes'
|
||||
* using repo "repo" repoid RepositoryId(0) (glob)
|
||||
processed 100
|
||||
* processed 5 (glob)
|
||||
|
||||
$ sqlite3 "$TESTTMP/monsql/sqlite_dbs" "SELECT HEX(filenode), HEX(linknode) FROM filenodes ORDER BY filenode DESC;"
|
||||
DDAE7A95B6B0FB27DFACC4051C41AA9CFF30C1E2|389710B7D0BE8C40323D3D96F717219F1CCE9601
|
||||
|
Loading…
Reference in New Issue
Block a user