From 553343bc0d1bfe436a7410dda2ef2f96df20efdd Mon Sep 17 00:00:00 2001 From: Stanislau Hlebik Date: Wed, 18 Oct 2017 01:34:45 -0700 Subject: [PATCH] mononoke: filter the same manifest entries in blobimporting Summary: There is no need to insert the same entries twice. Let's filter them. Note that while it's possible to have the same manifest entries (for example, file or dirs with the same content), all changeset entries should be unique, because each changeset in the repo is unique and is processed exactly once. Reviewed By: farnz Differential Revision: D6076667 fbshipit-source-id: 64bdf25a21884eb2faf43f32590f7cbb8f8dd300 --- cmds/blobimport.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/cmds/blobimport.rs b/cmds/blobimport.rs index e59cadd0ae..68c884528c 100644 --- a/cmds/blobimport.rs +++ b/cmds/blobimport.rs @@ -417,12 +417,20 @@ where let receiverstream = stream::iter_ok::<_, ()>(recv); let mut core = Core::new().expect("cannot create core in iothread"); let blobstore = open_blobstore(output, blobtype, &core.remote(), postpone_compaction)?; + // Filter only manifest entries, because changeset entries should be unique + let mut inserted_manifest_entries = std::collections::HashSet::new(); let stream = receiverstream .map(move |sender_helper| match sender_helper { BlobstoreEntry::Changeset(bcs) => { bcs.save(blobstore.clone()).from_err().boxify() } - BlobstoreEntry::ManifestEntry((key, value)) => blobstore.put(key, value), + BlobstoreEntry::ManifestEntry((key, value)) => { + if inserted_manifest_entries.insert(key.clone()) { + blobstore.put(key, value) + } else { + Ok(()).into_future().boxify() + } + }, }) .map_err(|_| Error::from("error happened")) .buffer_unordered(channel_size);