facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-05 14:28:17 +03:00
Code Issues Projects Releases Wiki Activity

Implement tag support in gitimport

Browse Source 
Summary:
This diff includes the following key changes:
- Added `generate_changeset_for_tag` method in the `GitUploader` contract for creating changesets for annotated tags pointing to commits
- Handled `BookmarkCategory` of tags and branches when creating the Mononoke bookmark for git branches or tags
- Introduced the `TagMetadata` type that captures relevant data from a git tag
- Implemented `generate_changeset_for_tag` for `DirectUploader` in `gitimport`
- Modified integration test to validate annotated tag handling during `gitimport`

Differential Revision: D45534203

fbshipit-source-id: 15d00b16831fab601226a85f86bd2c2b65968ae3
This commit is contained in:
Rajiv Sharma 2023-05-10 03:41:43 -07:00 committed by Facebook GitHub Bot
parent 1a4d9a756f
commit bcf031b194
5 changed files with 148 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
eden/mononoke/git/gitimport/src/main.rs
View File

@ -28,6 +28,7 @@ use clap::Subcommand;
use context::CoreContext;
use fbinit::FacebookInit;
use futures::future;
use import_tools::create_changeset_for_annotated_tag;
use import_tools::import_tree_as_single_bonsai_changeset;
use import_tools::upload_git_tag;
use import_tools::GitimportPreferences;
@ -35,6 +36,7 @@ use import_tools::GitimportTarget;
use linked_hash_map::LinkedHashMap;
use mercurial_derivation::get_manifest_from_bonsai;
use mercurial_derivation::DeriveHgChangeset;
use mononoke_api::BookmarkCategory;
use mononoke_api::BookmarkFreshness;
use mononoke_api::BookmarkKey;
use mononoke_app::args::RepoArgs;
@ -276,6 +278,7 @@ async fn async_main(app: MononokeApp) -> Result<(), Error> {
changeset.map(|cs| (maybe_tag_id, name, cs))
})
{
let final_changeset = changeset.clone();
let mut name = name
.strip_prefix("refs/")
.context("Ref does not start with refs/")?
@ -287,48 +290,58 @@ async fn async_main(app: MononokeApp) -> Result<(), Error> {
// Skip the HEAD revision: it shouldn't be imported as a bookmark in mononoke
continue;
}
// The ref getting imported is a tag, so store the raw git Tag object.
if let Some(tag_id) = maybe_tag_id {
// The ref getting imported is a tag, so store the raw git Tag object.
upload_git_tag(&ctx, &uploader, path, &prefs, tag_id).await?;
// Create the changeset corresponding to the commit pointed to by the tag.
create_changeset_for_annotated_tag(
&ctx, &uploader, path, &prefs, tag_id, changeset,
)
.await?;
}
// Set the appropriate category for branch and tag bookmarks
let bookmark_key = if maybe_tag_id.is_some() {
BookmarkKey::with_name_and_category(name.parse()?, BookmarkCategory::Tag)
} else {
BookmarkKey::new(&name)?
};
let pushvars = None;
if repo_context
.create_bookmark(&BookmarkKey::new(&name)?, *changeset, pushvars)
.create_bookmark(&bookmark_key, final_changeset, pushvars)
.await
.is_err()
{
// TODO (pierre): handle category here
let old_changeset = repo_context
.resolve_bookmark(&BookmarkKey::new(&name)?, BookmarkFreshness::MostRecent)
.resolve_bookmark(&bookmark_key, BookmarkFreshness::MostRecent)
.await
.with_context(|| format!("failed to resolve bookmark {name}"))?
.map(|context| context.id());
if old_changeset != Some(*changeset) {
if old_changeset != Some(final_changeset) {
let allow_non_fast_forward = true;
repo_context
.move_bookmark(
&BookmarkKey::new(&name)?,
*changeset,
&bookmark_key,
final_changeset,
old_changeset,
allow_non_fast_forward,
pushvars,
)
.await
.with_context(|| format!("failed to move bookmark {name} from {old_changeset:?} to {changeset:?}"))?;
.with_context(|| format!("failed to move bookmark {name} from {old_changeset:?} to {final_changeset:?}"))?;
info!(
ctx.logger(),
"Bookmark: \"{name}\": {changeset:?} (moved from {old_changeset:?})"
"Bookmark: \"{name}\": {final_changeset:?} (moved from {old_changeset:?})"
);
} else {
info!(
ctx.logger(),
"Bookmark: \"{name}\": {changeset:?} (already up-to-date)"
"Bookmark: \"{name}\": {final_changeset:?} (already up-to-date)"
);
}
} else {
info!(
ctx.logger(),
"Bookmark: \"{name}\": {changeset:?} (created)"
"Bookmark: \"{name}\": {final_changeset:?} (created)"
);
}
}

27
eden/mononoke/git/import_direct/src/uploader.rs
View File

@ -7,6 +7,7 @@
use std::sync::Arc;
use anyhow::Context;
use anyhow::Error;
use async_trait::async_trait;
use blobrepo::save_bonsai_changesets;
@ -25,10 +26,14 @@ use git_hash::ObjectId;
use import_tools::CommitMetadata;
use import_tools::GitImportLfs;
use import_tools::GitUploader;
use import_tools::TagMetadata;
use import_tools::HGGIT_COMMIT_ID_EXTRA;
use import_tools::HGGIT_MARKER_EXTRA;
use import_tools::HGGIT_MARKER_VALUE;
use mononoke_api::repo::git::create_annotated_tag;
use mononoke_api::repo::upload_git_object;
use mononoke_types::bonsai_changeset::BonsaiAnnotatedTag;
use mononoke_types::bonsai_changeset::BonsaiAnnotatedTagTarget;
use mononoke_types::hash;
use mononoke_types::BonsaiChangeset;
use mononoke_types::BonsaiChangesetMut;
@ -216,6 +221,28 @@ where
)
})
}
async fn generate_changeset_for_annotated_tag(
&self,
ctx: &CoreContext,
target_changeset_id: ChangesetId,
mut tag: TagMetadata,
) -> Result<ChangesetId, Error> {
let annotated_tag = BonsaiAnnotatedTag {
target: BonsaiAnnotatedTagTarget::Changeset(target_changeset_id),
pgp_signature: tag.pgp_signature.take(),
};
create_annotated_tag(
ctx,
&*self.inner,
tag.author.take(),
tag.author_date.take().map(|date| date.into()),
tag.message,
annotated_tag,
)
.await
.context("Failure in creating changeset for tag")
}
}
fn git_store_request(

63
eden/mononoke/git/import_tools/src/gitimport_objects.rs
View File

@ -27,6 +27,7 @@ use git_hash::ObjectId;
use git_object::bstr::BString;
use git_object::tree;
use git_object::Commit;
use git_object::Tag;
use git_object::Tree;
use manifest::Entry;
use manifest::Manifest;
@ -286,6 +287,51 @@ impl GitimportTarget {
}
}
#[derive(Debug)]
pub struct TagMetadata {
pub message: String,
pub author: Option<String>,
pub author_date: Option<DateTime>,
pub name: String,
pub pgp_signature: Option<Bytes>,
}
impl TagMetadata {
pub async fn new(
ctx: &CoreContext,
oid: ObjectId,
reader: &GitRepoReader,
) -> Result<Self, Error> {
let Tag {
name,
mut tagger,
message,
mut pgp_signature,
..
} = read_tag(reader, &oid).await?;
let author_date = tagger
.take()
.map(|tagger| convert_time_to_datetime(&tagger.time))
.transpose()?;
let author = tagger
.take()
.map(|tagger| format_signature(tagger.to_ref()));
let message = decode_message(&message, &None, ctx.logger())?;
let name = decode_message(&name, &None, ctx.logger())?;
let pgp_signature = pgp_signature
.take()
.map(|signature| Bytes::from(signature.to_vec()));
Result::<_, Error>::Ok(TagMetadata {
author,
author_date,
name,
message,
pgp_signature,
})
}
}
pub struct CommitMetadata {
pub oid: ObjectId,
pub parents: Vec<ObjectId>,
@ -304,6 +350,14 @@ pub struct ExtractedCommit {
pub original_commit: Bytes,
}
pub(crate) async fn read_tag(reader: &GitRepoReader, oid: &git_hash::oid) -> Result<Tag, Error> {
let object = reader.get_object(oid).await?;
object
.parsed
.try_into_tag()
.map_err(|_| format_err!("{} is not a tag", oid))
}
pub(crate) async fn read_commit(
reader: &GitRepoReader,
oid: &git_hash::oid,
@ -494,6 +548,15 @@ pub trait GitUploader: Clone + Send + Sync + 'static {
dry_run: bool,
) -> Result<(Self::IntermediateChangeset, ChangesetId), Error>;
/// Generate a single Bonsai changeset ID for corresponding Git
/// annotated tag.
async fn generate_changeset_for_annotated_tag(
&self,
ctx: &CoreContext,
target_changeset_id: ChangesetId,
tag: TagMetadata,
) -> Result<ChangesetId, Error>;
/// Finalize a batch of generated changesets. The supplied batch is
/// topologically sorted so that parents are all present before children
/// If you did not finalize the changeset in generate_changeset,

22
eden/mononoke/git/import_tools/src/lib.rs
View File

@ -55,6 +55,7 @@ pub use crate::gitimport_objects::GitTree;
pub use crate::gitimport_objects::GitUploader;
pub use crate::gitimport_objects::GitimportPreferences;
pub use crate::gitimport_objects::GitimportTarget;
pub use crate::gitimport_objects::TagMetadata;
pub use crate::gitlfs::GitImportLfs;
pub use crate::gitlfs::LfsMetaData;
@ -155,6 +156,27 @@ pub async fn is_annotated_tag(
.is_some())
}
pub async fn create_changeset_for_annotated_tag<Uploader: GitUploader>(
ctx: &CoreContext,
uploader: &Uploader,
path: &Path,
prefs: &GitimportPreferences,
tag_id: &ObjectId,
original_changeset_id: &ChangesetId,
) -> Result<ChangesetId, Error> {
let reader = GitRepoReader::new(&prefs.git_command_path, path).await?;
// Get the parsed Git Tag
let tag_metadata = TagMetadata::new(ctx, *tag_id, &reader)
.await
.with_context(|| format_err!("Failed to create TagMetadata from git tag {}", tag_id))?;
// Create the corresponding changeset for the Git Tag at Mononoke end
let changeset_id = uploader
.generate_changeset_for_annotated_tag(ctx, *original_changeset_id, tag_metadata)
.await
.with_context(|| format_err!("Failed to generate changeset for git tag {}", tag_id))?;
Ok(changeset_id)
}
pub async fn upload_git_tag<Uploader: GitUploader>(
ctx: &CoreContext,
uploader: &Uploader,

18
eden/mononoke/tests/integration/test-gitimport.t
View File

@ -42,7 +42,7 @@
* Ref: "refs/heads/master": Some(ChangesetId(Blake2(032cd4dce0406f1c1dd1362b6c3c9f9bdfa82f2fc5615e237a890be4fe08b044))) (glob)
* Ref: "refs/remotes/origin/HEAD": Some(ChangesetId(Blake2(032cd4dce0406f1c1dd1362b6c3c9f9bdfa82f2fc5615e237a890be4fe08b044))) (glob)
* Ref: "refs/remotes/origin/master": Some(ChangesetId(Blake2(032cd4dce0406f1c1dd1362b6c3c9f9bdfa82f2fc5615e237a890be4fe08b044))) (glob)
* Ref: "refs/tags/first_tag": Some(ChangesetId(Blake2(*))) (glob)
* Ref: "refs/tags/first_tag": Some(ChangesetId(Blake2(032cd4dce0406f1c1dd1362b6c3c9f9bdfa82f2fc5615e237a890be4fe08b044))) (glob)
# Validate if creating the commit also uploaded the raw commit blob AND the raw tree blob
# The ids of the blobs should be the same as the commit and tree object ids
@ -121,7 +121,7 @@
* Ref: "refs/heads/master": Some(ChangesetId(Blake2(*))) (glob)
* Ref: "refs/remotes/origin/HEAD": Some(ChangesetId(Blake2(032cd4dce0406f1c1dd1362b6c3c9f9bdfa82f2fc5615e237a890be4fe08b044))) (glob)
* Ref: "refs/remotes/origin/master": Some(ChangesetId(Blake2(032cd4dce0406f1c1dd1362b6c3c9f9bdfa82f2fc5615e237a890be4fe08b044))) (glob)
* Ref: "refs/tags/empty_tag": Some(ChangesetId(Blake2(*))) (glob)
* Ref: "refs/tags/empty_tag": Some(ChangesetId(Blake2(da93dc81badd8d407db0f3219ec0ec78f1ef750ebfa95735bb483310371af80c))) (glob)
* Ref: "refs/tags/first_tag": Some(ChangesetId(Blake2(032cd4dce0406f1c1dd1362b6c3c9f9bdfa82f2fc5615e237a890be4fe08b044))) (glob)
* Initializing repo: repo (glob)
* Initialized repo: repo (glob)
@ -129,7 +129,7 @@
* Bookmark: "heads/master": ChangesetId(Blake2(da93dc81badd8d407db0f3219ec0ec78f1ef750ebfa95735bb483310371af80c)) (created) (glob)
* Bookmark: "heads/master": ChangesetId(Blake2(032cd4dce0406f1c1dd1362b6c3c9f9bdfa82f2fc5615e237a890be4fe08b044)) (moved from Some(ChangesetId(Blake2(da93dc81badd8d407db0f3219ec0ec78f1ef750ebfa95735bb483310371af80c)))) (glob)
* Bookmark: "tags/empty_tag": ChangesetId(Blake2(da93dc81badd8d407db0f3219ec0ec78f1ef750ebfa95735bb483310371af80c)) (created) (glob)
* Bookmark: "tags/first_tag": ChangesetId(Blake2(*)) (created) (glob)
* Bookmark: "tags/first_tag": ChangesetId(Blake2(032cd4dce0406f1c1dd1362b6c3c9f9bdfa82f2fc5615e237a890be4fe08b044)) (created) (glob)
# Generating bookmarks should upload the raw tag object to blobstore.
# The id of the blob should be the same as the tag object id
@ -148,6 +148,12 @@
8963e1f55d1346a07c3aec8c8fc72bf87d0452b1
fb02ed046a1e75fe2abb8763f7c715496ae36353
# Generating bookmarks should also create the changeset corresponding to the
# git tag at Mononoke end
$ ls $TESTTMP/blobstore/blobs | grep -e d5be6fdf77fc73ee5e3a4bab1adbb4772829e06c0f104e6cc0d70cabf1ebff4b -e 5ca579c0e3ebea708371b65ce559e5a51b231ad1b6f3cdfd874ca27362a2a6a8
blob-repo0000.changeset.blake2.5ca579c0e3ebea708371b65ce559e5a51b231ad1b6f3cdfd874ca27362a2a6a8
blob-repo0000.changeset.blake2.d5be6fdf77fc73ee5e3a4bab1adbb4772829e06c0f104e6cc0d70cabf1ebff4b
# Importing a second time should still work
$ gitimport "$GIT_REPO" --generate-bookmarks full-repo
* using repo "repo" repoid RepositoryId(0) (glob)
@ -164,7 +170,7 @@
* Bookmark: "heads/master": ChangesetId(Blake2(da93dc81badd8d407db0f3219ec0ec78f1ef750ebfa95735bb483310371af80c)) (moved from Some(ChangesetId(Blake2(032cd4dce0406f1c1dd1362b6c3c9f9bdfa82f2fc5615e237a890be4fe08b044)))) (glob)
* Bookmark: "heads/master": ChangesetId(Blake2(032cd4dce0406f1c1dd1362b6c3c9f9bdfa82f2fc5615e237a890be4fe08b044)) (moved from Some(ChangesetId(Blake2(da93dc81badd8d407db0f3219ec0ec78f1ef750ebfa95735bb483310371af80c)))) (glob)
* Bookmark: "tags/empty_tag": ChangesetId(Blake2(da93dc81badd8d407db0f3219ec0ec78f1ef750ebfa95735bb483310371af80c)) (already up-to-date) (glob)
* Bookmark: "tags/first_tag": ChangesetId(Blake2(*)) (already up-to-date) (glob)
* Bookmark: "tags/first_tag": ChangesetId(Blake2(032cd4dce0406f1c1dd1362b6c3c9f9bdfa82f2fc5615e237a890be4fe08b044)) (already up-to-date) (glob)
# Start Mononoke
@ -190,5 +196,5 @@
# Checks all the bookmarks were created
$ hg bookmarks --all
* heads/master * (glob)
* tags/empty_tag * (glob)
* tags/first_tag * (glob)
* tags/empty_tag * e7f52161c612 (glob)
* tags/first_tag * b48ed4600785 (glob)