facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-09 00:14:35 +03:00
Code Issues Projects Releases Wiki Activity

mononoke: fix gitimport to use correct filestore config

Browse Source 
Summary:
I believe this is the reason for -
https://fb.workplace.com/groups/238845853462687/posts/845939069420026. We used
default config that doesn't do any chunking and puts large files as a single
blobs.

Let's not do that

Reviewed By: farnz

Differential Revision: D31209331

fbshipit-source-id: 43c2d2ab7caac110a1474856da09c119a5e72429
This commit is contained in:
Stanislau Hlebik 2021-09-27 11:16:31 -07:00 committed by Facebook GitHub Bot
parent 48cfd2e7ee
commit 5ef97baefb
2 changed files with 63 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
eden/mononoke/git/import_tools/src/lib.rs
View File

@ -86,6 +86,7 @@ async fn lfs_store_request(
async fn do_upload<B: Blobstore + Clone + 'static>(
ctx: &CoreContext,
blobstore: &B,
filestore_config: FilestoreConfig,
pool: GitPool,
oid: Oid,
path: &MPath,
@ -110,7 +111,7 @@ async fn do_upload<B: Blobstore + Clone + 'static>(
(req, bstream.right_stream())
};
let meta = filestore::store(blobstore, FilestoreConfig::default(), ctx, &req, bstream).await?;
let meta = filestore::store(blobstore, filestore_config, ctx, &req, bstream).await?;
Ok(meta)
}
@ -119,6 +120,7 @@ async fn do_upload<B: Blobstore + Clone + 'static>(
async fn find_file_changes<S, B: Blobstore + Clone + 'static>(
ctx: &CoreContext,
blobstore: &B,
filestore_config: &FilestoreConfig,
pool: GitPool,
changes: S,
lfs: &GitImportLfs,
@ -129,12 +131,21 @@ where
changes
.map_ok(|change| async {
task::spawn({
cloned!(pool, ctx, blobstore, lfs);
cloned!(pool, ctx, blobstore, filestore_config, lfs);
async move {
match change {
BonsaiDiffFileChange::Changed(path, ty, GitLeaf(oid))
| BonsaiDiffFileChange::ChangedReusedId(path, ty, GitLeaf(oid)) => {
let meta = do_upload(&ctx, &blobstore, pool, oid, &path, &lfs).await?;
let meta = do_upload(
&ctx,
&blobstore,
filestore_config,
pool,
oid,
&path,
&lfs,
)
.await?;
Ok((
path,
FileChange::tracked(meta.content_id, ty, meta.total_size, None),
@ -237,6 +248,7 @@ pub async fn gitimport_acc<Acc: GitimportAccumulator>(
let file_changes = find_file_changes(
&ctx,
repo.blobstore(),
&repo.filestore_config(),
pool.clone(),
bonsai_diff(ctx.clone(), pool, tree, parent_trees),
&lfs,
@ -540,8 +552,16 @@ pub async fn import_tree_as_single_bonsai_changeset(
cloned!(lfs);
async move {
let path = MPath::new(path)?;
let content_metadata =
do_upload(&ctx, repo.blobstore(), pool.clone(), oid, &path, &lfs).await?;
let content_metadata = do_upload(
&ctx,
repo.blobstore(),
repo.filestore_config(),
pool.clone(),
oid,
&path,
&lfs,
)
.await?;
let file_type = convert_git_filemode(mode)?;
let file_change = FileChange::tracked(
content_metadata.content_id,

38
eden/mononoke/tests/integration/test-gitimport-filestore.t Normal file
View File

@ -0,0 +1,38 @@
# Copyright (c) Facebook, Inc. and its affiliates.
#
# This software may be used and distributed according to the terms of the
# GNU General Public License found in the LICENSE file in the root
# directory of this source tree.
$ FILESTORE=1
$ FILESTORE_CHUNK_SIZE=10
$ . "${TEST_FIXTURES}/library.sh"
$ ENABLED_DERIVED_DATA='["git_trees", "filenodes", "hgchangesets"]' setup_common_config
$ GIT_REPO="${TESTTMP}/repo-git"
$ HG_REPO="${TESTTMP}/repo-hg"
# Setup git repsitory
$ mkdir "$GIT_REPO"
$ cd "$GIT_REPO"
$ git init -q
$ echo "laaaaaaaaaarge file" > file1
$ git add file1
$ git commit -am "Add file1"
[master (root-commit) 0ecc922] Add file1
1 file changed, 1 insertion(+)
create mode 100644 file1
# Import it into Mononoke
$ cd "$TESTTMP"
$ gitimport "$GIT_REPO" --derive-trees --derive-hg --hggit-compatibility --bonsai-git-mapping full-repo
* using repo "repo" repoid RepositoryId(0) (glob)
*Reloading redacted config from configerator* (glob)
* GitRepo:*repo-git commit 1 of 1 - Oid:* => Bid:* (glob)
* 1 tree(s) are valid! (glob)
* Hg: 0ecc922af7b11d796a715f3c093673914b060164: HgManifestId(HgNodeHash(Sha1(4f16e4ceeccf36b18e4a72e183c16a9bea650e1d))) (glob)
* Ref: Some("refs/heads/master"): Some(ChangesetId(Blake2(306010a179d752b69fa2dd068a3fab8077043c217d3fb7370f705a50acbef48c))) (glob)
$ mononoke_admin filestore is-chunked id 48ef00ac63821b09154b55f1b380d253f936afb076a873e1bcc1d137c8b5bab2
* using repo "repo" repoid RepositoryId(0) (glob)
* Reloading redacted config from configerator (glob)
chunked