2019-10-11 23:51:17 +03:00
|
|
|
/*
|
|
|
|
* Copyright (c) Facebook, Inc. and its affiliates.
|
|
|
|
*
|
|
|
|
* This software may be used and distributed according to the terms of the
|
2020-02-11 13:42:43 +03:00
|
|
|
* GNU General Public License version 2.
|
2019-10-11 23:51:17 +03:00
|
|
|
*/
|
2019-08-21 12:22:32 +03:00
|
|
|
|
2019-12-07 03:26:57 +03:00
|
|
|
use anyhow::{Error, Result};
|
2019-08-21 12:22:32 +03:00
|
|
|
use bytes::Bytes;
|
|
|
|
use clap::Arg;
|
2020-01-22 03:10:07 +03:00
|
|
|
use cmdlib::{args, helpers::block_execute};
|
2019-08-21 12:22:32 +03:00
|
|
|
use context::CoreContext;
|
2019-09-14 06:16:08 +03:00
|
|
|
use fbinit::FacebookInit;
|
rust: Rename futures_preview:: to futures::
Summary:
Context: https://fb.workplace.com/groups/rust.language/permalink/3338940432821215/
This codemod replaces *all* dependencies on `//common/rust/renamed:futures-preview` with `fbsource//third-party/rust:futures-preview` and their uses in Rust code from `futures_preview::` to `futures::`.
This does not introduce any collisions with `futures::` meaning 0.1 futures because D20168958 previously renamed all of those to `futures_old::` in crates that depend on *both* 0.1 and 0.3 futures.
Codemod performed by:
```
rg \
--files-with-matches \
--type-add buck:TARGETS \
--type buck \
--glob '!/experimental' \
--regexp '(_|\b)rust(_|\b)' \
| sed 's,TARGETS$,:,' \
| xargs \
-x \
buck query "labels(srcs, rdeps(%Ss, //common/rust/renamed:futures-preview, 1))" \
| xargs sed -i 's,\bfutures_preview::,futures::,'
rg \
--files-with-matches \
--type-add buck:TARGETS \
--type buck \
--glob '!/experimental' \
--regexp '(_|\b)rust(_|\b)' \
| xargs sed -i 's,//common/rust/renamed:futures-preview,fbsource//third-party/rust:futures-preview,'
```
Reviewed By: k21
Differential Revision: D20213432
fbshipit-source-id: 07ee643d350c5817cda1f43684d55084f8ac68a6
2020-03-03 21:58:44 +03:00
|
|
|
use futures::{
|
2020-01-28 20:15:36 +03:00
|
|
|
compat::Future01CompatExt,
|
|
|
|
future::TryFutureExt,
|
|
|
|
stream::{self, TryStreamExt},
|
|
|
|
};
|
rust: Rename futures_preview:: to futures::
Summary:
Context: https://fb.workplace.com/groups/rust.language/permalink/3338940432821215/
This codemod replaces *all* dependencies on `//common/rust/renamed:futures-preview` with `fbsource//third-party/rust:futures-preview` and their uses in Rust code from `futures_preview::` to `futures::`.
This does not introduce any collisions with `futures::` meaning 0.1 futures because D20168958 previously renamed all of those to `futures_old::` in crates that depend on *both* 0.1 and 0.3 futures.
Codemod performed by:
```
rg \
--files-with-matches \
--type-add buck:TARGETS \
--type buck \
--glob '!/experimental' \
--regexp '(_|\b)rust(_|\b)' \
| sed 's,TARGETS$,:,' \
| xargs \
-x \
buck query "labels(srcs, rdeps(%Ss, //common/rust/renamed:futures-preview, 1))" \
| xargs sed -i 's,\bfutures_preview::,futures::,'
rg \
--files-with-matches \
--type-add buck:TARGETS \
--type buck \
--glob '!/experimental' \
--regexp '(_|\b)rust(_|\b)' \
| xargs sed -i 's,//common/rust/renamed:futures-preview,fbsource//third-party/rust:futures-preview,'
```
Reviewed By: k21
Differential Revision: D20213432
fbshipit-source-id: 07ee643d350c5817cda1f43684d55084f8ac68a6
2020-03-03 21:58:44 +03:00
|
|
|
use futures_ext::FutureExt;
|
2019-08-21 12:22:32 +03:00
|
|
|
use lfs_import_lib::lfs_upload;
|
2019-08-27 14:21:31 +03:00
|
|
|
use mercurial_types::blobs::File;
|
2019-08-21 12:22:32 +03:00
|
|
|
|
|
|
|
const NAME: &str = "lfs_import";
|
|
|
|
|
|
|
|
const ARG_LFS_HELPER: &str = "lfs-helper";
|
|
|
|
const ARG_CONCURRENCY: &str = "concurrency";
|
|
|
|
const ARG_POINTERS: &str = "pointers";
|
2019-12-10 14:59:21 +03:00
|
|
|
const ARG_NO_CREATE: &str = "no-create";
|
2019-08-21 12:22:32 +03:00
|
|
|
|
|
|
|
const DEFAULT_CONCURRENCY: usize = 16;
|
|
|
|
|
2019-09-14 06:16:08 +03:00
|
|
|
#[fbinit::main]
|
|
|
|
fn main(fb: FacebookInit) -> Result<()> {
|
2019-10-28 17:57:34 +03:00
|
|
|
let app = args::MononokeApp::new(NAME)
|
|
|
|
.with_advanced_args_hidden()
|
|
|
|
.build()
|
|
|
|
.version("0.0.0")
|
|
|
|
.about("Import LFS blobs")
|
|
|
|
.arg(
|
|
|
|
Arg::with_name(ARG_CONCURRENCY)
|
|
|
|
.long("concurrency")
|
|
|
|
.takes_value(true)
|
|
|
|
.help("The number of OIDs to process in parallel"),
|
|
|
|
)
|
2019-12-10 14:59:21 +03:00
|
|
|
.arg(
|
|
|
|
Arg::with_name(ARG_NO_CREATE)
|
|
|
|
.long(ARG_NO_CREATE)
|
|
|
|
.takes_value(false)
|
|
|
|
.required(false)
|
|
|
|
.help("If provided won't create a new repo"),
|
|
|
|
)
|
2019-10-28 17:57:34 +03:00
|
|
|
.arg(
|
|
|
|
Arg::with_name(ARG_LFS_HELPER)
|
|
|
|
.required(true)
|
|
|
|
.takes_value(true)
|
|
|
|
.help("LFS Helper"),
|
|
|
|
)
|
|
|
|
.arg(
|
|
|
|
Arg::with_name(ARG_POINTERS)
|
|
|
|
.takes_value(true)
|
|
|
|
.required(true)
|
|
|
|
.min_values(1)
|
|
|
|
.help("Raw LFS pointers to be imported"),
|
|
|
|
);
|
2019-08-21 12:22:32 +03:00
|
|
|
|
|
|
|
let matches = app.get_matches();
|
2020-01-31 20:50:17 +03:00
|
|
|
args::init_cachelib(fb, &matches, None);
|
2019-08-21 12:22:32 +03:00
|
|
|
|
2019-10-13 10:44:52 +03:00
|
|
|
let logger = args::init_logging(fb, &matches);
|
2019-09-14 06:16:08 +03:00
|
|
|
let ctx = CoreContext::new_with_logger(fb, logger.clone());
|
2019-12-10 14:59:21 +03:00
|
|
|
let blobrepo = if matches.is_present(ARG_NO_CREATE) {
|
|
|
|
args::open_repo(fb, &logger, &matches).left_future()
|
|
|
|
} else {
|
|
|
|
args::create_repo(fb, &logger, &matches).right_future()
|
|
|
|
};
|
2019-08-21 12:22:32 +03:00
|
|
|
let lfs_helper = matches.value_of(ARG_LFS_HELPER).unwrap().to_string();
|
|
|
|
|
|
|
|
let concurrency: usize = matches
|
|
|
|
.value_of(ARG_CONCURRENCY)
|
|
|
|
.map_or(Ok(DEFAULT_CONCURRENCY), |j| j.parse())
|
|
|
|
.map_err(Error::from)?;
|
|
|
|
|
2020-01-28 20:15:36 +03:00
|
|
|
let entries: Vec<_> = matches
|
2019-08-21 12:22:32 +03:00
|
|
|
.values_of(ARG_POINTERS)
|
|
|
|
.unwrap()
|
|
|
|
.into_iter()
|
mononoke: update internals to Bytes 0.5
Summary:
The Bytes 0.5 update left us in a somewhat undesirable position where every
access to our blobstore incurs an extra copy whenever we fetch data out of our
cache (by turning it from Bytes 0.5 into Bytes 0.4) — we also have quite a few
place where we convert in one direction then immediately into the other.
Internally, we can start using Bytes 0.5 now. For example, this is useful when
pulling data out of our blobstore and deserializing as Thrift (or conversely,
when serializing and putting it into our blobstore).
However, when we interface with Tokio (i.e. decoders & encoders), we still have
to use Bytes 0.4. So, when needed, we convert our Bytes 0.5 to 0.4 there.
The tradeoff idea is that we deal with more bytes internally than we end up
sending to clients, so doing the Bytes conversion closer to the point of
sending data to clients means less copies.
We can also start removing those once we migrate to Tokio 0.2 (and newer
versions of Hyper for HTTP services).
Changes that were required:
- You can't extend new bytes (because that implicitly copies). You need to use
BytesMut instead, which I did where that was necessary (I also added calls in
the Filestore to do that efficiently).
- You can't create bytes from a `&'a [u8]`, unless `'a` is `'static`. You need
to use `copy_from_slice` instead.
- `slice_to` and `slice_from` have been replaced by a `slice()` function that
takes ranges.
Reviewed By: StanislavGlebik
Differential Revision: D20121350
fbshipit-source-id: eb31af2051fd8c9d31c69b502e2f6f1ce2190cb1
2020-02-27 19:04:39 +03:00
|
|
|
.map(|e| File::new(Bytes::copy_from_slice(e.as_bytes()), None, None).get_lfs_content())
|
2019-08-21 12:22:32 +03:00
|
|
|
.collect();
|
|
|
|
|
2020-01-28 20:15:36 +03:00
|
|
|
let import = async move {
|
|
|
|
let blobrepo = blobrepo.compat().await?;
|
|
|
|
stream::iter(entries)
|
|
|
|
.try_for_each_concurrent(concurrency, |lfs| {
|
|
|
|
lfs_upload(ctx.clone(), blobrepo.clone(), lfs_helper.clone(), lfs)
|
|
|
|
.compat()
|
|
|
|
.map_ok(|_| ())
|
|
|
|
})
|
|
|
|
.await
|
|
|
|
};
|
2019-08-21 12:22:32 +03:00
|
|
|
|
2020-01-29 20:39:49 +03:00
|
|
|
block_execute(
|
|
|
|
import,
|
|
|
|
fb,
|
|
|
|
NAME,
|
|
|
|
&logger,
|
|
|
|
&matches,
|
|
|
|
cmdlib::monitoring::AliveService,
|
|
|
|
)
|
2019-08-21 12:22:32 +03:00
|
|
|
}
|