mononoke/repo_import: add rewrite_commit functionality

Summary:
Previous commit: D22233127 (fa1caa8c4e)

In this diff, I added rewrite commit path functionality using Mover https://fburl.com/diffusion/6rnf9q2f to repo_import.
Given a prefix (e.g. new_repo), we prepend the paths of the files extracted from the bonsaichangesets given by gitimport (e.g. folder1/file1 => new_repo/folder1/file1). Previously, we did this manually when importing a git repo (https://www.internalfb.com/intern/wiki/Mercurial/Admin/ImportingRepos/) using convert extension.

Reviewed By: StanislavGlebik

Differential Revision: D22307039

fbshipit-source-id: 322533e5d6cbaf5d7eec589c8cba0c1b9c79d7af

eden/mononoke/commit_rewriting/movers/src/lib.rs

@@ -70,7 +70,7 @@ enum PathAction {
 
 /// Default action to apply to a path when syncing between two repos
 #[derive(Debug, Clone)]
-enum DefaultAction {
+pub enum DefaultAction {
     /// Prepend path with this prefix
     PrependPrefix(MPath),
     /// Keep the path as is
@@ -167,7 +167,7 @@ where
 }
 
 /// Create a `Mover`, given a path prefix map and a default action
-fn mover_factory(
+pub fn mover_factory(
     prefix_map: HashMap<MPath, PrefixAction>,
     default_action: DefaultAction,
 ) -> Result<Mover> {

eden/mononoke/repo_import/Cargo.toml

@@ -7,13 +7,16 @@ license = "GPLv2+"
 include = ["src/**/*.rs"]
 
 [dependencies]
+blobrepo = { path = "../blobrepo" }
 cmdlib = { path = "../cmdlib" }
 context = { path = "../server/context" }
+cross_repo_sync = { path = "../commit_rewriting/cross_repo_sync" }
 import_tools = { path = "../git/import_tools" }
+mercurial_types = { path = "../mercurial/types" }
 mononoke_types = { path = "../mononoke_types" }
+movers = { path = "../commit_rewriting/movers" }
 fbinit = { git = "https://github.com/facebookexperimental/rust-shed.git", branch = "master" }
 anyhow = "1.0"
 clap = "2.33"
 futures = { version = "0.3", features = ["async-await", "compat"] }
-git2 = "0.13"
-linked-hash-map = { version = "0.5", features = ["serde_impl"] }
+slog = { version = "2.5", features = ["max_level_debug"] }

eden/mononoke/repo_import/src/main.rs

@@ -6,19 +6,69 @@
  */
 
 use anyhow::Error;
+use blobrepo::{save_bonsai_changesets, BlobRepo};
 use clap::Arg;
 use cmdlib::args;
 use cmdlib::helpers::block_execute;
 use context::CoreContext;
+use cross_repo_sync::rewrite_commit;
 use fbinit::FacebookInit;
 use futures::compat::Future01CompatExt;
-use git2::Oid;
 use import_tools::{GitimportPreferences, GitimportTarget};
-use linked_hash_map::LinkedHashMap;
-use mononoke_types::{BonsaiChangeset, ChangesetId};
+use mercurial_types::MPath;
+use mononoke_types::ChangesetId;
+use movers::DefaultAction;
+use slog::info;
+use std::collections::HashMap;
 use std::path::Path;
 
 const ARG_GIT_REPOSITORY_PATH: &str = "git-repository-path";
+const ARG_DEST_PATH_PREFIX: &str = "destination-path-prefix";
+
+async fn rewrite_file_paths(
+    ctx: &CoreContext,
+    repo: &BlobRepo,
+    path: &Path,
+    prefix: &str,
+) -> Result<(), Error> {
+    let prefs = GitimportPreferences::default();
+    let target = GitimportTarget::FullRepo;
+    let import_map = import_tools::gitimport(ctx, repo, path, target, prefs).await?;
+    let mut remapped_parents: HashMap<ChangesetId, ChangesetId> = HashMap::new();
+    let mover = movers::mover_factory(
+        HashMap::new(),
+        DefaultAction::PrependPrefix(MPath::new(prefix).unwrap()),
+    )?;
+    let mut bonsai_changesets = vec![];
+
+    for (_id, (bcs_id, bcs)) in import_map {
+        let bcs_mut = bcs.into_mut();
+        let rewritten_bcs_opt = rewrite_commit(
+            ctx.clone(),
+            bcs_mut,
+            &remapped_parents,
+            mover.clone(),
+            repo.clone(),
+        )
+        .await?;
+
+        if let Some(rewritten_bcs_mut) = rewritten_bcs_opt {
+            let rewritten_bcs = rewritten_bcs_mut.freeze()?;
+            remapped_parents.insert(bcs_id, rewritten_bcs.get_changeset_id());
+            info!(
+                ctx.logger(),
+                "Remapped {:?} => {:?}",
+                bcs_id,
+                rewritten_bcs.get_changeset_id(),
+            );
+            bonsai_changesets.push(rewritten_bcs);
+        }
+    }
+    save_bonsai_changesets(bonsai_changesets.clone(), ctx.clone(), repo.clone())
+        .compat()
+        .await?;
+    Ok(())
+}
 
 #[fbinit::main]
 fn main(fb: FacebookInit) -> Result<(), Error> {
@@ -27,35 +77,37 @@ fn main(fb: FacebookInit) -> Result<(), Error> {
         .build()
         .version("0.0.0")
         .about("Automating repository imports")
-        .arg(Arg::with_name(ARG_GIT_REPOSITORY_PATH).help("Path to a git repository to import"));
+        .arg(
+            Arg::with_name(ARG_GIT_REPOSITORY_PATH)
+                .required(true)
+                .help("Path to a git repository to import"),
+        )
+        .arg(
+            Arg::with_name(ARG_DEST_PATH_PREFIX)
+                .long(ARG_DEST_PATH_PREFIX)
+                .required(true)
+                .takes_value(true)
+                .help("Prefix of the destination folder we import to"),
+        );
 
-    let prefs = GitimportPreferences::default();
     let matches = app.get_matches();
 
-    let target = GitimportTarget::FullRepo;
-
     let path = Path::new(matches.value_of(ARG_GIT_REPOSITORY_PATH).unwrap());
+    let prefix = matches.value_of(ARG_DEST_PATH_PREFIX).unwrap();
     args::init_cachelib(fb, &matches, None);
 
     let logger = args::init_logging(fb, &matches);
     let ctx = CoreContext::new_with_logger(fb, logger.clone());
     let repo = args::create_repo(fb, &logger, &matches);
-
-    let gitimport_result: Result<LinkedHashMap<Oid, (ChangesetId, BonsaiChangeset)>, Error> =
-        block_execute(
-            async {
-                let repo = repo.compat().await?;
-                import_tools::gitimport(&ctx, &repo, &path, target, prefs).await
-            },
-            fb,
-            "gitimport",
-            &logger,
-            &matches,
-            cmdlib::monitoring::AliveService,
-        );
-
-    match gitimport_result {
-        Ok(_import_map) => Ok(()),
-        Err(e) => Err(e),
-    }
+    block_execute(
+        async {
+            let repo = repo.compat().await?;
+            rewrite_file_paths(&ctx, &repo, &path, &prefix).await
+        },
+        fb,
+        "gitimport",
+        &logger,
+        &matches,
+        cmdlib::monitoring::AliveService,
+    )
 }

eden/mononoke/tests/integration/test-repo_import.t

@@ -15,24 +15,39 @@
   $ git init
   Initialized empty Git repository in $TESTTMP/repo-git/.git/
   $ echo "this is file1" > file1
-  $ git add file1
-  $ git commit -am "Add file1"
-  [master (root-commit) 8ce3eae] Add file1
-   1 file changed, 1 insertion(+)
+  $ mkdir file2_repo
+  $ cd file2_repo
+  $ echo "this is file2" > file2
+  $ cd ..
+  $ git add file1 file2_repo/file2
+  $ git commit -am "Add file1 and file2"
+  [master (root-commit) ce435b0] Add file1 and file2
+   2 files changed, 2 insertions(+)
    create mode 100644 file1
+   create mode 100644 file2_repo/file2
+  $ mkdir file3_repo
+  $ echo "this is file3" > file3_repo/file3
+  $ git add file3_repo/file3
+  $ git commit -am "Add file3"
+  [master 2c01e4a] Add file3
+   1 file changed, 1 insertion(+)
+   create mode 100644 file3_repo/file3
 
 # Import it into Mononoke
   $ cd "$TESTTMP"
-  $ repo_import "$GIT_REPO"
+  $ repo_import "$GIT_REPO" --destination-path-prefix "new_repo"
   * using repo "repo" repoid RepositoryId(0) (glob)
-  * Created 8ce3eae44760b500bf3f2c3922a95dcd3c908e9e => ChangesetId(Blake2(22e96bdf5536bb09049caadadaa0cd9ea0593556db6fa389fa044f8ee95270ea)) (glob)
-  * 1 bonsai changesets have been committed (glob)
-  * Ref: Some("refs/heads/master"): Some(ChangesetId(Blake2(22e96bdf5536bb09049caadadaa0cd9ea0593556db6fa389fa044f8ee95270ea))) (glob)
+  * Created ce435b03d4ef526648f8654c61e26ae5cc1069cc => ChangesetId(Blake2(f7cbf75d9c08ff96896ed2cebd0327aa514e58b1dd9901d50129b9e08f4aa062)) (glob)
+  * Created 2c01e4a5658421e2bfcd08e31d9b69399319bcd3 => ChangesetId(Blake2(f7708ed066b1c23591f862148e0386ec704a450e572154cc52f87ca0e394a0fb)) (glob)
+  * 2 bonsai changesets have been committed (glob)
+  * Ref: Some("refs/heads/master"): Some(ChangesetId(Blake2(f7708ed066b1c23591f862148e0386ec704a450e572154cc52f87ca0e394a0fb))) (glob)
+  * Remapped ChangesetId(Blake2(f7cbf75d9c08ff96896ed2cebd0327aa514e58b1dd9901d50129b9e08f4aa062)) => ChangesetId(Blake2(4c9a9394cb65d5b57286d866bb012a0a4553ea05ba82755c0ed9e977e51d0da0)) (glob)
+  * Remapped ChangesetId(Blake2(f7708ed066b1c23591f862148e0386ec704a450e572154cc52f87ca0e394a0fb)) => ChangesetId(Blake2(cf29a57c1ba299f835ea2546e26c8eb8fd5b981067162579924b58000306e96f)) (glob)
 
 # Set master (gitimport does not do this yet)
-  $ mononoke_admin bookmarks set master 22e96bdf5536bb09049caadadaa0cd9ea0593556db6fa389fa044f8ee95270ea
+  $ mononoke_admin bookmarks set master cf29a57c1ba299f835ea2546e26c8eb8fd5b981067162579924b58000306e96f
   * using repo "repo" repoid RepositoryId(0) (glob)
-  * changeset resolved as: ChangesetId(Blake2(22e96bdf5536bb09049caadadaa0cd9ea0593556db6fa389fa044f8ee95270ea)) (glob)
+  * changeset resolved as: ChangesetId(Blake2(cf29a57c1ba299f835ea2546e26c8eb8fd5b981067162579924b58000306e96f)) (glob)
   * Current position of BookmarkName { bookmark: "master" } is None (glob)
 
 # Start Mononoke
@@ -43,5 +58,9 @@
   $ cd "$TESTTMP"
   $ hgmn_clone 'ssh://user@dummy/repo' "$HG_REPO"
   $ cd "$HG_REPO"
-  $ cat "file1"
+  $ cat "new_repo/file1"
   this is file1
+  $ cat "new_repo/file2_repo/file2"
+  this is file2
+  $ cat "new_repo/file3_repo/file3"
+  this is file3