From 521388fdd59f14d166b4b8931618b31e4c7b6667 Mon Sep 17 00:00:00 2001 From: Alex Hornby Date: Tue, 10 Dec 2019 03:59:21 -0800 Subject: [PATCH] mononoke: separate blobstore naming from repo naming in integration tests Summary: Allow repos to share blobstore in tests by naming blobstore storage_config separately from the repo using storage.toml Reviewed By: krallin Differential Revision: D18820758 fbshipit-source-id: b37159b1407481dc69f548fad08cc543d1bbc34f --- cmdlib/src/args.rs | 10 +- cmdlib/src/helpers.rs | 27 +++-- cmds/lfs_import.rs | 15 ++- tests/integration/library.sh | 100 ++++++++++++------ tests/integration/test-alias-verify.t | 10 +- tests/integration/test-blobimport-no-create.t | 51 +++++++++ tests/integration/test-bundle2-preservation.t | 6 +- tests/integration/test-hash-validation.t | 2 +- .../test-lfs-upload-alias-on-fetch.t | 6 +- .../integration/test-multiplexed-blobstore.t | 2 +- tests/integration/test-rechunker.t | 6 +- 11 files changed, 170 insertions(+), 65 deletions(-) create mode 100644 tests/integration/test-blobimport-no-create.t diff --git a/cmdlib/src/args.rs b/cmdlib/src/args.rs index f88c467855..62cf62539d 100644 --- a/cmdlib/src/args.rs +++ b/cmdlib/src/args.rs @@ -33,7 +33,7 @@ use slog_logview::LogViewDrain; use crate::helpers::{ init_cachelib_from_settings, open_sql_with_config_and_myrouter_port, setup_repo_dir, - CachelibSettings, + CachelibSettings, CreateStorage, }; use crate::log; @@ -767,7 +767,13 @@ fn open_repo_internal_with_repo_id<'a>( info!(logger, "using repo \"{}\" repoid {:?}", reponame, repo_id); match &config.storage_config.blobstore { BlobConfig::Files { path } | BlobConfig::Rocks { path } | BlobConfig::Sqlite { path } => { - setup_repo_dir(path, create).expect("Setting up file blobrepo failed"); + let create = if create { + // Many path repos can share one blobstore, so allow store to exist or create it. + CreateStorage::ExistingOrCreate + } else { + CreateStorage::ExistingOnly + }; + try_boxfuture!(setup_repo_dir(path, create)); } _ => {} }; diff --git a/cmdlib/src/helpers.rs b/cmdlib/src/helpers.rs index 9922fc106e..e4e944c6cb 100644 --- a/cmdlib/src/helpers.rs +++ b/cmdlib/src/helpers.rs @@ -53,13 +53,20 @@ pub fn upload_and_show_trace(ctx: CoreContext) -> impl Future>(data_dir: P, create: bool) -> Result<()> { +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum CreateStorage { + ExistingOnly, + ExistingOrCreate, +} + +pub fn setup_repo_dir>(data_dir: P, create: CreateStorage) -> Result<()> { let data_dir = data_dir.as_ref(); if !data_dir.is_dir() { bail!("{:?} does not exist or is not a directory", data_dir); } + // Validate directory layout for subdir in &["blobs"] { let subdir = data_dir.join(subdir); @@ -67,20 +74,12 @@ pub fn setup_repo_dir>(data_dir: P, create: bool) -> Result<()> { bail!("{:?} already exists and is not a directory", subdir); } - if create { - if subdir.exists() { - let content: Vec<_> = subdir.read_dir()?.collect(); - if !content.is_empty() { - bail!( - "{:?} already exists and is not empty: {:?}", - subdir, - content - ); - } - } else { - fs::create_dir(&subdir) - .with_context(|| format!("failed to create subdirectory {:?}", subdir))?; + if !subdir.exists() { + if CreateStorage::ExistingOnly == create { + bail!("{:?} not found in ExistingOnly mode", subdir,); } + fs::create_dir(&subdir) + .with_context(|| format!("failed to create subdirectory {:?}", subdir))?; } } Ok(()) diff --git a/cmds/lfs_import.rs b/cmds/lfs_import.rs index b224fce4d1..b9a2c7e50c 100644 --- a/cmds/lfs_import.rs +++ b/cmds/lfs_import.rs @@ -13,6 +13,7 @@ use cmdlib::{args, helpers::create_runtime}; use context::CoreContext; use fbinit::FacebookInit; use futures::{stream, Future, IntoFuture, Stream}; +use futures_ext::FutureExt; use lfs_import_lib::lfs_upload; use mercurial_types::blobs::File; @@ -21,6 +22,7 @@ const NAME: &str = "lfs_import"; const ARG_LFS_HELPER: &str = "lfs-helper"; const ARG_CONCURRENCY: &str = "concurrency"; const ARG_POINTERS: &str = "pointers"; +const ARG_NO_CREATE: &str = "no-create"; const DEFAULT_CONCURRENCY: usize = 16; @@ -37,6 +39,13 @@ fn main(fb: FacebookInit) -> Result<()> { .takes_value(true) .help("The number of OIDs to process in parallel"), ) + .arg( + Arg::with_name(ARG_NO_CREATE) + .long(ARG_NO_CREATE) + .takes_value(false) + .required(false) + .help("If provided won't create a new repo"), + ) .arg( Arg::with_name(ARG_LFS_HELPER) .required(true) @@ -56,7 +65,11 @@ fn main(fb: FacebookInit) -> Result<()> { let logger = args::init_logging(fb, &matches); let ctx = CoreContext::new_with_logger(fb, logger.clone()); - let blobrepo = args::open_repo(fb, &logger, &matches); + let blobrepo = if matches.is_present(ARG_NO_CREATE) { + args::open_repo(fb, &logger, &matches).left_future() + } else { + args::create_repo(fb, &logger, &matches).right_future() + }; let lfs_helper = matches.value_of(ARG_LFS_HELPER).unwrap().to_string(); let concurrency: usize = matches diff --git a/tests/integration/library.sh b/tests/integration/library.sh index a75f8d763b..2d5d6fdbc3 100644 --- a/tests/integration/library.sh +++ b/tests/integration/library.sh @@ -8,16 +8,8 @@ # Library routines and initial setup for Mononoke-related tests. if [[ -n "$DB_SHARD_NAME" ]]; then - function db_config() { - echo "[storage.blobstore.db.remote]" - echo "db_address=\"$DB_SHARD_NAME\"" - } MONONOKE_DEFAULT_START_TIMEOUT=60 else - function db_config() { - echo "[storage.blobstore.db.local]" - echo "local_db_path=\"$TESTTMP/monsql\"" - } MONONOKE_DEFAULT_START_TIMEOUT=15 fi @@ -401,6 +393,12 @@ function setup_mononoke_config { REPOTYPE="blob_rocks" if [[ $# -gt 0 ]]; then REPOTYPE="$1" + shift + fi + local blobstorename=blobstore + if [[ $# -gt 0 ]]; then + blobstorename="$1" + shift fi if [[ ! -e "$TESTTMP/mononoke_hgcli" ]]; then @@ -423,7 +421,49 @@ identity_type = "USER" identity_data = "$ALLOWED_USERNAME" CONFIG - setup_mononoke_repo_config "$REPONAME" + echo "# Start new config" > common/storage.toml + setup_mononoke_storage_config "$REPOTYPE" "$blobstorename" + + setup_mononoke_repo_config "$REPONAME" "$blobstorename" +} + +function db_config() { + local blobstorename="$1" + if [[ -n "$DB_SHARD_NAME" ]]; then + echo "[$blobstorename.db.remote]" + echo "db_address=\"$DB_SHARD_NAME\"" + else + echo "[$blobstorename.db.local]" + echo "local_db_path=\"$TESTTMP/monsql\"" + fi +} + +function setup_mononoke_storage_config { + local underlyingstorage="$1" + local blobstorename="$2" + local blobstorepath="$TESTTMP/$blobstorename" + + if [[ -v MULTIPLEXED ]]; then + mkdir -p "$blobstorepath/0" "$blobstorepath/1" + cat >> common/storage.toml <> common/storage.toml < "repos/$reponame/server.toml" <> "repos/$reponame/server.toml" <> "repos/$reponame/server.toml" <> "repos/$reponame/server.toml" <> "repos/$reponame/server.toml" <> "repos/$reponame/server.toml" <> "$TESTTMP/blobimport.out" 2>&1 + "$input" "${CACHING_ARGS[@]}" "$@" > "$TESTTMP/blobimport.out" 2>&1 BLOBIMPORT_RC="$?" if [[ $BLOBIMPORT_RC -ne 0 ]]; then cat "$TESTTMP/blobimport.out" # set exit code, otherwise previous cat sets it to 0 return "$BLOBIMPORT_RC" + elif [[ -n "$always_log" ]]; then + cat "$TESTTMP/blobimport.out" fi } diff --git a/tests/integration/test-alias-verify.t b/tests/integration/test-alias-verify.t index 3f07793d64..079b1a1532 100644 --- a/tests/integration/test-alias-verify.t +++ b/tests/integration/test-alias-verify.t @@ -25,7 +25,7 @@ $ blobimport repo-hg-nolfs/.hg repo - $ ls $TESTTMP/repo/blobs | grep "alias" + $ ls $TESTTMP/blobstore/blobs | grep "alias" blob-repo0000.alias.gitsha1.45d9e0e9fc8859787c33081dffdf12f41b54fcf3 blob-repo0000.alias.gitsha1.8e1e71d5ce34c01b6fe83bc5051545f2918c8c2b blob-repo0000.alias.gitsha1.9de77c18733ab8009a956c25e28c85fe203a17d7 @@ -41,8 +41,8 @@ * Alias Verification finished: 0 errors found (glob) - $ rm -rf $TESTTMP/repo/blobs/blob-repo0000.alias.* - $ ls $TESTTMP/repo/blobs | grep "alias" | wc -l + $ rm -rf $TESTTMP/blobstore/blobs/blob-repo0000.alias.* + $ ls $TESTTMP/blobstore/blobs | grep "alias" | wc -l 0 $ aliasverify verify 2>&1 | grep "Alias Verification" @@ -54,7 +54,7 @@ * Missing alias blob: alias Sha256(d690916cdea320e620748799a2051a0f4e07d6d0c3e2bc199ea3c69e0c0b5e4f), content_id ContentId(Blake2(7ee06cac57ab4267c097ebc8ec36e903fb3c25867934fe360e069ea1ab2ed7fd)) (glob) * Missing alias blob: alias Sha256(2ba85baaa7922ff4c0dfdbc00fd07bd69dcb1dce745c6a8c676fe8b5642a0d66), content_id ContentId(Blake2(1a3f1094cdae123ec6999b7baf4211ffd94f47970bedd71e13ec07f24a9aba6a)) (glob) - $ ls $TESTTMP/repo/blobs | grep "alias" | wc -l + $ ls $TESTTMP/blobstore/blobs | grep "alias" | wc -l 0 $ aliasverify generate --debug 2>&1 | grep "Missing alias blob" @@ -62,7 +62,7 @@ * Missing alias blob: alias Sha256(d690916cdea320e620748799a2051a0f4e07d6d0c3e2bc199ea3c69e0c0b5e4f), content_id ContentId(Blake2(7ee06cac57ab4267c097ebc8ec36e903fb3c25867934fe360e069ea1ab2ed7fd)) (glob) * Missing alias blob: alias Sha256(2ba85baaa7922ff4c0dfdbc00fd07bd69dcb1dce745c6a8c676fe8b5642a0d66), content_id ContentId(Blake2(1a3f1094cdae123ec6999b7baf4211ffd94f47970bedd71e13ec07f24a9aba6a)) (glob) - $ ls $TESTTMP/repo/blobs | grep "alias" + $ ls $TESTTMP/blobstore/blobs | grep "alias" blob-repo0000.alias.sha256.2ba85baaa7922ff4c0dfdbc00fd07bd69dcb1dce745c6a8c676fe8b5642a0d66 blob-repo0000.alias.sha256.b9a294f298d0ed2b65ca4488a42b473ff5f75d0b9843cbea84e1b472f9a514d1 blob-repo0000.alias.sha256.d690916cdea320e620748799a2051a0f4e07d6d0c3e2bc199ea3c69e0c0b5e4f diff --git a/tests/integration/test-blobimport-no-create.t b/tests/integration/test-blobimport-no-create.t new file mode 100644 index 0000000000..6a5625b9f4 --- /dev/null +++ b/tests/integration/test-blobimport-no-create.t @@ -0,0 +1,51 @@ + $ . "${TEST_FIXTURES}/library.sh" + +setup configuration + $ setup_common_config + $ cd $TESTTMP + +setup common configuration + $ cat >> $HGRCPATH < [ui] + > ssh="$DUMMYSSH" + > EOF + +setup repo + $ hg init repo-hg + $ cd repo-hg + $ setup_hg_server + $ hg debugdrawdag < C + > | + > B + > | + > A + > EOF + +create master bookmark + $ hg bookmark master_bookmark -r tip + +blobimport --no-create with no storage present, should fail due to missing directory + $ cd .. + $ blobimport --log repo-hg/.hg repo --no-create + * using repo "repo" repoid RepositoryId(0)* (glob) + Error: "$TESTTMP/blobstore/blobs" not found in ExistingOnly mode + [1] + +blobimport, succeeding, creates directory if not existing + $ blobimport --log repo-hg/.hg repo + * using repo "repo" repoid RepositoryId(0) (glob) + * inserted commits # 0 (glob) + * finished uploading changesets and globalrevs (glob) + * uploaded chunk of 1 bookmarks (glob) + +check the bookmark is there after import + $ mononoke_admin --readonly-storage bookmarks log master_bookmark 2>&1 | grep master_bookmark + (master_bookmark) 26805aba1e600a82e93661149f2313866a221a7b blobimport * (glob) + +blobimport --no-create after successful import, should be fine as storage shared with previous good run + $ blobimport --log repo-hg/.hg repo --no-create + * using repo "repo" repoid RepositoryId(0) (glob) + * inserted commits # 0 (glob) + * finished uploading changesets and globalrevs (glob) + * uploaded chunk of 0 bookmarks (glob) diff --git a/tests/integration/test-bundle2-preservation.t b/tests/integration/test-bundle2-preservation.t index 74fe5a8724..a1c95d1083 100644 --- a/tests/integration/test-bundle2-preservation.t +++ b/tests/integration/test-bundle2-preservation.t @@ -50,7 +50,7 @@ Create a new commit in repo2 Do a push, while bundle preservation is disabled $ hgmn push -qr . --to master_bookmark - $ ls $TESTTMP/repo/blobs | grep rawbundle2 + $ ls $TESTTMP/blobstore/blobs | grep rawbundle2 [1] Restart mononoke with enabled bundle2 preservation @@ -71,7 +71,7 @@ Do a push, while bundle preservation is enabled searching for changes updating bookmark master_bookmark - $ ls $TESTTMP/repo/blobs | grep rawbundle2 | wc -l + $ ls $TESTTMP/blobstore/blobs | grep rawbundle2 | wc -l 1 Do a pushrebase, while preservation is enabled @@ -84,5 +84,5 @@ Do a pushrebase, while preservation is enabled adding file changes added 1 changesets with 0 changes to 0 files updating bookmark master_bookmark - $ ls $TESTTMP/repo/blobs | grep rawbundle2 | wc -l + $ ls $TESTTMP/blobstore/blobs | grep rawbundle2 | wc -l 2 diff --git a/tests/integration/test-hash-validation.t b/tests/integration/test-hash-validation.t index a9aba69bdb..aa96ab46f9 100644 --- a/tests/integration/test-hash-validation.t +++ b/tests/integration/test-hash-validation.t @@ -38,7 +38,7 @@ blobimport them into Mononoke storage and start Mononoke $ blobimport repo-hg/.hg repo Corrupt blobs by replacing one content blob with another - $ cd repo/blobs + $ cd blobstore/blobs $ cp blob-repo0000.content.blake2.896ad5879a5df0403bfc93fc96507ad9c93b31b11f3d0fa05445da7918241e5d blob-repo0000.content.blake2.eb56488e97bb4cf5eb17f05357b80108a4a71f6c3bab52dfcaec07161d105ec9 start mononoke diff --git a/tests/integration/test-lfs-upload-alias-on-fetch.t b/tests/integration/test-lfs-upload-alias-on-fetch.t index 7818f5fa17..7e4a12cbac 100644 --- a/tests/integration/test-lfs-upload-alias-on-fetch.t +++ b/tests/integration/test-lfs-upload-alias-on-fetch.t @@ -74,7 +74,7 @@ updating bookmark master_bookmark # Check that alias.sha1.hgfilenode -> sha256.file_content is not generated - $ ls $TESTTMP/repo/blobs | grep "alias" | sort -h + $ ls $TESTTMP/blobstore/blobs | grep "alias" | sort -h blob-repo0000.alias.gitsha1.23c160a91fd3e722a49a86017e103e83e7965af7 blob-repo0000.alias.gitsha1.b4785957bc986dc39c629de9fac9df46972c00fc blob-repo0000.alias.sha1.8cfc11d4c1bf45aca9412afc5b95cfd1db83e885 @@ -106,7 +106,7 @@ added 1 changesets with 0 changes to 0 files new changesets 0db8825b9792 - $ ls $TESTTMP/repo/blobs | grep "alias.content" | wc -l + $ ls $TESTTMP/blobstore/blobs | grep "alias.content" | wc -l 0 $ hgmn update -r master_bookmark -v @@ -119,7 +119,7 @@ 2 files updated, 0 files merged, 0 files removed, 0 files unresolved # Check that alias.sha1.hgfilenode -> sha256.file_content is generated - $ ls $TESTTMP/repo/blobs | grep "alias" | sort -h + $ ls $TESTTMP/blobstore/blobs | grep "alias" | sort -h blob-repo0000.alias.gitsha1.23c160a91fd3e722a49a86017e103e83e7965af7 blob-repo0000.alias.gitsha1.b4785957bc986dc39c629de9fac9df46972c00fc blob-repo0000.alias.sha1.8cfc11d4c1bf45aca9412afc5b95cfd1db83e885 diff --git a/tests/integration/test-multiplexed-blobstore.t b/tests/integration/test-multiplexed-blobstore.t index 2686b4d2f2..40ce408874 100644 --- a/tests/integration/test-multiplexed-blobstore.t +++ b/tests/integration/test-multiplexed-blobstore.t @@ -1,7 +1,7 @@ $ . "${TEST_FIXTURES}/library.sh" setup configuration - $ MULTIPLEXED=1 setup_common_config + $ MULTIPLEXED=1 REPOTYPE=blob_files setup_common_config $ cd $TESTTMP setup common configuration diff --git a/tests/integration/test-rechunker.t b/tests/integration/test-rechunker.t index 01ce725b41..d9c2ada9fd 100644 --- a/tests/integration/test-rechunker.t +++ b/tests/integration/test-rechunker.t @@ -20,13 +20,13 @@ $ cd "$TESTTMP" $ blobimport repo-hg/.hg repo - $ FILENODE=$(ls "$TESTTMP/repo/blobs" | grep "hgfilenode" | cut -d "." -f 4) + $ FILENODE=$(ls "$TESTTMP/blobstore/blobs" | grep "hgfilenode" | cut -d "." -f 4) # Check that nothing happens if the filestore is not enabled $ mononoke_rechunker "$FILENODE" * using repo "repo" repoid RepositoryId(0) (glob) - $ ls "$TESTTMP/repo/blobs" | grep hgfilenode + $ ls "$TESTTMP/blobstore/blobs" | grep hgfilenode blob-repo0000.hgfilenode.sha1.92c09d364cd563132d6eb5f1424ff63523d51f73 # Check that the rechunker complains about an unknown filenode @@ -43,5 +43,5 @@ $ mononoke_rechunker "$FILENODE" * using repo "repo" repoid RepositoryId(0) (glob) - $ ls "$TESTTMP/repo/blobs" | grep chunk | wc -l + $ ls "$TESTTMP/blobstore/blobs" | grep chunk | wc -l 2