From 126a661d8c53e233f67209dbe2cfee6afc04751e Mon Sep 17 00:00:00 2001 From: Viet Hung Nguyen Date: Thu, 13 Aug 2020 00:41:03 -0700 Subject: [PATCH] mononoke/repo_import: add commit push functionality Summary: After creating the merge commit (D23028163 (https://github.com/facebookexperimental/eden/commit/f267bec3f7cd38fee30647c3b9afe362db5c8d0c)) from the imported commit head and the destination bookmark's head, we need to push the commit onto that bookmark. This diff adds the push functionality to repo_import tool. Note: GlobalrevPushrebaseHook is a hook to assign globalrevs to commits to keep the order of the commits Reviewed By: StanislavGlebik Differential Revision: D23072966 fbshipit-source-id: ff815467ed0f96de86da3de9a628fd45743eb167 --- eden/mononoke/repo_import/Cargo.toml | 5 + eden/mononoke/repo_import/src/main.rs | 117 ++++++++++++++++-- .../tests/integration/test-repo_import.t | 32 ++++- 3 files changed, 142 insertions(+), 12 deletions(-) diff --git a/eden/mononoke/repo_import/Cargo.toml b/eden/mononoke/repo_import/Cargo.toml index 7ff860db94..8ae7258710 100644 --- a/eden/mononoke/repo_import/Cargo.toml +++ b/eden/mononoke/repo_import/Cargo.toml @@ -18,14 +18,18 @@ derived_data_utils = { path = "../derived_data/utils" } import_tools = { path = "../git/import_tools" } manifest = { path = "../manifest" } mercurial_types = { path = "../mercurial/types" } +metaconfig_types = { path = "../metaconfig/types" } mononoke_types = { path = "../mononoke_types" } movers = { path = "../commit_rewriting/movers" } mutable_counters = { path = "../mutable_counters" } +pushrebase = { path = "../pushrebase" } topo_sort = { path = "../common/topo_sort" } +unbundle = { path = "../repo_client/unbundle" } fbinit = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" } anyhow = "1.0" clap = "2.33" futures = { version = "0.3.5", features = ["async-await", "compat"] } +maplit = "1.0" serde = { version = "1.0", features = ["derive", "rc"] } serde_json = "1.0" slog = { version = "2.5", features = ["max_level_debug"] } @@ -37,4 +41,5 @@ mercurial_types-mocks = { path = "../mercurial/types/mocks" } mononoke_types-mocks = { path = "../mononoke_types/mocks" } sql_construct = { path = "../common/sql_construct" } tests_utils = { path = "../tests/utils" } +sql = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" } tokio-compat = "0.1" diff --git a/eden/mononoke/repo_import/src/main.rs b/eden/mononoke/repo_import/src/main.rs index dd1804643e..15ac747d1d 100644 --- a/eden/mononoke/repo_import/src/main.rs +++ b/eden/mononoke/repo_import/src/main.rs @@ -24,10 +24,13 @@ use futures::{ }; use import_tools::{GitimportPreferences, GitimportTarget}; use manifest::ManifestOps; +use maplit::hashset; use mercurial_types::{HgChangesetId, MPath}; +use metaconfig_types::RepoConfig; use mononoke_types::{BonsaiChangeset, BonsaiChangesetMut, ChangesetId, DateTime}; use movers::DefaultAction; use mutable_counters::{MutableCounters, SqlMutableCounters}; +use pushrebase::{do_pushrebase_bonsai, OntoBookmarkParams}; use serde::{Deserialize, Serialize}; use serde_json; use slog::info; @@ -37,6 +40,7 @@ use std::num::NonZeroUsize; use std::path::Path; use tokio::{fs, io::AsyncWriteExt, process, time}; use topo_sort::sort_topological; +use unbundle::get_pushrebase_hooks; mod cli; @@ -250,15 +254,15 @@ async fn merge_imported_commit( ctx: &CoreContext, repo: &BlobRepo, shifted_bcs: &[BonsaiChangeset], - dest_bookmark: &str, + dest_bookmark: &BookmarkName, changeset_args: ChangesetArgs, -) -> Result<(), Error> { +) -> Result { info!( ctx.logger(), "Merging the imported commits into given bookmark, {}", dest_bookmark ); let master_cs_id = match repo - .get_bonsai_bookmark(ctx.clone(), &BookmarkName::new(dest_bookmark)?) + .get_bonsai_bookmark(ctx.clone(), dest_bookmark) .compat() .await? { @@ -325,7 +329,39 @@ async fn merge_imported_commit( .compat() .await?; info!(ctx.logger(), "Finished merging"); - Ok(()) + Ok(merged_cs_id) +} + +async fn push_merge_commit( + ctx: &CoreContext, + repo: &BlobRepo, + merged_cs_id: ChangesetId, + bookmark_to_merge_into: &BookmarkName, + repo_config: &RepoConfig, +) -> Result { + info!(ctx.logger(), "Running pushrebase"); + + let merged_cs = merged_cs_id.load(ctx.clone(), repo.blobstore()).await?; + let pushrebase_flags = repo_config.pushrebase.flags; + let pushrebase_hooks = get_pushrebase_hooks(&repo, &repo_config.pushrebase); + + let pushrebase_res = do_pushrebase_bonsai( + &ctx, + &repo, + &pushrebase_flags, + &OntoBookmarkParams::new(bookmark_to_merge_into.clone()), + &hashset![merged_cs], + &None, + &pushrebase_hooks, + ) + .await?; + + let pushrebase_cs_id = pushrebase_res.head; + info!( + ctx.logger(), + "Finished pushrebasing to {}", pushrebase_cs_id + ); + Ok(pushrebase_cs_id) } async fn get_leaf_entries( @@ -527,7 +563,7 @@ fn main(fb: FacebookInit) -> Result<(), Error> { let sleep_time = matches.value_of(ARG_SLEEP_TIME).unwrap(); let sleep_time = sleep_time.parse::()?; let backup_hashes_path = matches.value_of(ARG_BACKUP_HASHES_FILE_PATH).unwrap(); - let dest_bookmark = matches.value_of(ARG_DEST_BOOKMARK).unwrap(); + let dest_bookmark_name = matches.value_of(ARG_DEST_BOOKMARK).unwrap(); let commit_author = matches.value_of(ARG_COMMIT_AUTHOR).unwrap(); let commit_message = matches.value_of(ARG_COMMIT_MESSAGE).unwrap(); let datetime = match matches.value_of(ARG_COMMIT_DATE_RFC3339) { @@ -567,7 +603,13 @@ fn main(fb: FacebookInit) -> Result<(), Error> { &mutable_counters, ) .await?; - merge_imported_commit(&ctx, &repo, &shifted_bcs, &dest_bookmark, changeset_args).await + let dest_bookmark = BookmarkName::new(dest_bookmark_name)?; + let merged_cs_id = + merge_imported_commit(&ctx, &repo, &shifted_bcs, &dest_bookmark, changeset_args) + .await?; + let (_, repo_config) = args::get_config_by_repoid(ctx.fb, &matches, repo.get_repoid())?; + push_merge_commit(&ctx, &repo, merged_cs_id, &dest_bookmark, &repo_config).await?; + Ok(()) }, fb, "repo_import", @@ -580,7 +622,8 @@ fn main(fb: FacebookInit) -> Result<(), Error> { #[cfg(test)] mod tests { use crate::{ - check_dependent_systems, move_bookmark, sort_bcs, CheckerFlags, LATEST_REPLAYED_REQUEST_KEY, + check_dependent_systems, merge_imported_commit, move_bookmark, push_merge_commit, sort_bcs, + ChangesetArgs, CheckerFlags, LATEST_REPLAYED_REQUEST_KEY, }; use anyhow::Result; @@ -590,11 +633,17 @@ mod tests { use fbinit::FacebookInit; use futures::{compat::Future01CompatExt, stream::TryStreamExt}; use mercurial_types_mocks::nodehash::ONES_CSID as HG_CSID; + use metaconfig_types::{PushrebaseParams, RepoConfig}; + use mononoke_types::{ + globalrev::{Globalrev, START_COMMIT_GLOBALREV}, + DateTime, RepositoryId, + }; use mononoke_types_mocks::changesetid::{ONES_CSID as MON_CSID, TWOS_CSID}; use mutable_counters::{MutableCounters, SqlMutableCounters}; + use sql::rusqlite::Connection as SqliteConnection; use sql_construct::SqlConstruct; use std::time::Duration; - use tests_utils::drawdag::create_from_dag; + use tests_utils::{bookmark, drawdag::create_from_dag, CreateCommitContext}; use tokio::time; fn create_bookmark_name(book: &str) -> BookmarkName { @@ -770,4 +819,56 @@ mod tests { .await?; Ok(()) } + + #[fbinit::compat_test] + async fn merge_push_commit_test(fb: FacebookInit) -> Result<()> { + let ctx = CoreContext::test_mock(fb); + let (repo, _con) = blobrepo_factory::new_memblob_with_sqlite_connection_with_id( + SqliteConnection::open_in_memory()?, + RepositoryId::new(1), + )?; + + let master_cs_id = CreateCommitContext::new_root(&ctx, &repo) + .add_file("a", "a") + .commit() + .await?; + let imported_cs_id = CreateCommitContext::new_root(&ctx, &repo) + .add_file("b", "b") + .commit() + .await?; + let imported_cs = imported_cs_id.load(ctx.clone(), repo.blobstore()).await?; + + let dest_bookmark = bookmark(&ctx, &repo, "master").set_to(master_cs_id).await?; + + let changeset_args = ChangesetArgs { + author: "user".to_string(), + message: "merging".to_string(), + datetime: DateTime::now(), + }; + + let merged_cs_id = merge_imported_commit( + &ctx, + &repo, + &[imported_cs.clone()], + &dest_bookmark, + changeset_args, + ) + .await?; + + let mut repo_config = RepoConfig::default(); + repo_config.pushrebase = PushrebaseParams { + assign_globalrevs: true, + ..Default::default() + }; + + let pushed_cs_id = + push_merge_commit(&ctx, &repo, merged_cs_id, &dest_bookmark, &repo_config).await?; + let pushed_cs = pushed_cs_id.load(ctx.clone(), repo.blobstore()).await?; + + assert_eq!( + Globalrev::new(START_COMMIT_GLOBALREV), + Globalrev::from_bcs(&pushed_cs)? + ); + Ok(()) + } } diff --git a/eden/mononoke/tests/integration/test-repo_import.t b/eden/mononoke/tests/integration/test-repo_import.t index 9fedfbb678..ec064ee3c9 100644 --- a/eden/mononoke/tests/integration/test-repo_import.t +++ b/eden/mononoke/tests/integration/test-repo_import.t @@ -69,6 +69,8 @@ * Creating a merge bonsai changeset with parents: *, * (glob) * Created merge bonsai: * and changeset: * (glob) * Finished merging (glob) + * Running pushrebase (glob) + * Finished pushrebasing to * (glob) # Check if we derived all the types $ BOOKMARK_NAME="repo_import_new_repo" @@ -106,11 +108,33 @@ adding changesets adding manifests adding file changes - added 2 changesets with 0 changes to 0 files + added 3 changesets with 0 changes to 0 files + updating bookmark master_bookmark adding remote bookmark repo_import_new_repo - $ hgmn up repo_import_new_repo - 3 files updated, 0 files merged, 0 files removed, 0 files unresolved - (activating bookmark repo_import_new_repo) + $ hgmn up master_bookmark + 6 files updated, 0 files merged, 0 files removed, 0 files unresolved + (activating bookmark master_bookmark) + + $ log -r "all()" + @ merging [draft;rev=5;*] (glob) + |\ + | o Add file3 [draft;rev=4;12e9a7555b29] + | | + | o Add file1 and file2 [draft;rev=3;25f978935fdd] + | + o C [draft;rev=2;26805aba1e60] + | + o B [draft;rev=1;112478962961] + | + o A [draft;rev=0;426bada5c675] + $ + + $ ls + A + B + C + new_dir + $ cat "new_dir/new_repo/file1" this is file1 $ cat "new_dir/new_repo/file2_repo/file2"