generic store|fetch for mononoke types

Summary:
Implementation of generic `store|fetch` for bonsai types.
 - bonsai types have unique typed hashes associated with each bonsai type, I'm leveraging this fact to implement generic `store|fetch` methods on `BlobRepo`

Reviewed By: farnz

Differential Revision: D8254810

fbshipit-source-id: 5f798fade4cb8d1ac851f94c7ad7e64636bbca65

blobrepo/src/errors.rs

@@ -43,6 +43,7 @@ pub type Result<T> = ::std::result::Result<T, Error>;
 
 #[derive(Debug, Fail)]
 pub enum ErrorKind {
+    #[fail(display = "Missing typed key entry for key: {}", _0)] MissingTypedKeyEntry(String),
     #[fail(display = "Error while opening state for {}", _0)] StateOpen(StateOpenError),
     #[fail(display = "Changeset id {} is missing", _0)] ChangesetMissing(HgChangesetId),
     #[fail(display = "Manifest id {} is missing", _0)] ManifestMissing(HgNodeHash),

blobrepo/src/repo.rs

@@ -36,8 +36,8 @@ use mercurial_types::{Changeset, Entry, HgBlob, HgBlobNode, HgChangesetId, HgFil
                       HgNodeHash, HgParents, Manifest, RepoPath, RepositoryId, Type};
 use mercurial_types::manifest;
 use mercurial_types::nodehash::HgManifestId;
-use mononoke_types::{Blob, BlobstoreBytes, BonsaiChangeset, ContentId, DateTime, FileChange,
-                     FileContents, MPath, MononokeId};
+use mononoke_types::{Blob, BlobstoreBytes, BlobstoreValue, BonsaiChangeset, ContentId, DateTime,
+                     FileChange, FileContents, MPath, MononokeId};
 use rocksblob::Rocksblob;
 use rocksdb;
 use tokio_core::reactor::Core;
@@ -239,6 +239,48 @@ impl BlobRepo {
         ))
     }
 
+    fn fetch<K>(&self, key: &K) -> impl Future<Item = K::Value, Error = Error> + Send
+    where
+        K: MononokeId,
+    {
+        let blobstore_key = key.blobstore_key();
+        self.blobstore
+            .get(blobstore_key.clone())
+            .and_then(move |blob| {
+                blob.ok_or(ErrorKind::MissingTypedKeyEntry(blobstore_key).into())
+                    .and_then(|blob| <<K as MononokeId>::Value>::from_blob(blob.into()))
+            })
+    }
+
+    // this is supposed to be used only from unittest
+    pub fn unittest_fetch<K>(&self, key: &K) -> impl Future<Item = K::Value, Error = Error> + Send
+    where
+        K: MononokeId,
+    {
+        self.fetch(key)
+    }
+
+    fn store<K, V>(&self, value: V) -> impl Future<Item = K, Error = Error> + Send
+    where
+        V: BlobstoreValue<Key = K>,
+        K: MononokeId<Value = V>,
+    {
+        let blob = value.into_blob();
+        let key = *blob.id();
+        self.blobstore
+            .put(key.blobstore_key(), blob.into())
+            .map(move |_| key)
+    }
+
+    // this is supposed to be used only from unittest
+    pub fn unittest_store<K, V>(&self, value: V) -> impl Future<Item = K, Error = Error> + Send
+    where
+        V: BlobstoreValue<Key = K>,
+        K: MononokeId<Value = V>,
+    {
+        self.store(value)
+    }
+
     pub fn get_file_content(&self, key: &HgNodeHash) -> BoxFuture<FileContents, Error> {
         fetch_file_content_and_renames_from_blobstore(&self.blobstore, *key)
             .map(|contentrename| contentrename.0)

blobrepo/test/main.rs

@@ -14,6 +14,7 @@ extern crate futures;
 extern crate futures_ext;
 #[macro_use]
 extern crate maplit;
+extern crate quickcheck;
 #[macro_use]
 extern crate slog;
 
@@ -28,11 +29,13 @@ extern crate mononoke_types;
 
 use futures::Future;
 use futures_ext::FutureExt;
+use quickcheck::{quickcheck, Arbitrary, Gen, TestResult, Testable};
+use std::marker::PhantomData;
 
 use blobrepo::{compute_changed_files, BlobRepo};
 use mercurial_types::{manifest, Changeset, Entry, FileType, HgChangesetId, HgEntryId,
                       HgManifestId, MPath, MPathElement, RepoPath};
-use mononoke_types::FileContents;
+use mononoke_types::{ChangesetId, ContentId, FileContents, MononokeId};
 
 mod stats_units;
 #[macro_use]
@@ -369,6 +372,50 @@ test_both_repotypes!(
     check_linknode_creation_eager
 );
 
+struct StoreFetchTestable<K> {
+    repo: BlobRepo,
+    _key: PhantomData<K>,
+}
+
+impl<K> StoreFetchTestable<K> {
+    fn new(repo: &BlobRepo) -> Self {
+        StoreFetchTestable {
+            repo: repo.clone(),
+            _key: PhantomData,
+        }
+    }
+}
+
+impl<K> Testable for StoreFetchTestable<K>
+where
+    K: MononokeId,
+    K::Value: PartialEq + Arbitrary,
+{
+    fn result<G: Gen>(&self, g: &mut G) -> TestResult {
+        let value = <K::Value as Arbitrary>::arbitrary(g);
+        let value_cloned = value.clone();
+        let store_fetch_future = self.repo
+            .unittest_store(value)
+            .and_then({
+                let repo = self.repo.clone();
+                move |key| repo.unittest_fetch(&key)
+            })
+            .map(move |value_fetched| TestResult::from_bool(value_fetched == value_cloned));
+        run_future(store_fetch_future).expect("valid mononoke type")
+    }
+}
+
+fn store_fetch_mononoke_types(repo: BlobRepo) {
+    quickcheck(StoreFetchTestable::<ChangesetId>::new(&repo));
+    quickcheck(StoreFetchTestable::<ContentId>::new(&repo));
+}
+
+test_both_repotypes!(
+    store_fetch_mononoke_types,
+    store_fetch_mononoke_types_lazy,
+    store_fetch_mononoke_types_eager
+);
+
 #[test]
 fn test_compute_changed_files_no_parents() {
     async_unit::tokio_unit_test(|| {

bundle2-resolver/src/changegroup/filelog.rs

@@ -22,6 +22,7 @@ use mercurial::file;
 use mercurial_bundles::changegroup::CgDeltaChunk;
 use mercurial_types::{delta, manifest, Delta, FileType, HgBlob, HgNodeHash, HgNodeKey, MPath,
                       RepoPath, NULL_HASH};
+use mononoke_types::BlobstoreValue;
 
 use errors::*;
 use stats::*;

mononoke-types/src/blob.rs

@@ -10,6 +10,7 @@ use bytes::Bytes;
 
 use asyncmemo::Weight;
 
+use errors::*;
 use typed_hash::{ChangesetId, ContentId, MononokeId};
 
 /// A serialized blob in memory.
@@ -95,3 +96,9 @@ impl<Id> From<Blob<Id>> for BlobstoreBytes {
         BlobstoreBytes::from_bytes(blob.data)
     }
 }
+
+pub trait BlobstoreValue: Sized + Send {
+    type Key;
+    fn into_blob(self) -> Blob<Self::Key>;
+    fn from_blob(Blob<Self::Key>) -> Result<Self>;
+}

mononoke-types/src/bonsai_changeset.rs

@@ -11,7 +11,7 @@ use quickcheck::{Arbitrary, Gen};
 
 use rust_thrift::compact_protocol;
 
-use blob::{Blob, ChangesetBlob};
+use blob::{Blob, BlobstoreValue, ChangesetBlob};
 use datetime::DateTime;
 use errors::*;
 use file_change::FileChange;
@@ -111,14 +111,6 @@ impl BonsaiChangeset {
         })?)
     }
 
-    pub fn from_blob<T: AsRef<[u8]>>(t: T) -> Result<Self> {
-        // TODO (T27336549) stop using SyncFailure once thrift is converted to failure
-        let thrift_tc = compact_protocol::deserialize(t.as_ref())
-            .map_err(SyncFailure::new)
-            .context(ErrorKind::BlobDeserializeError("BonsaiChangeset".into()))?;
-        Self::from_thrift(thrift_tc)
-    }
-
     /// Get the parents for this changeset. The order of parents is significant.
     pub fn parents(&self) -> impl Iterator<Item = &ChangesetId> {
         self.inner.parents.iter()
@@ -178,16 +170,6 @@ impl BonsaiChangeset {
         self.inner
     }
 
-    /// Serialize this structure into a blob.
-    pub fn into_blob(self) -> ChangesetBlob {
-        let thrift = self.into_thrift();
-        let data = compact_protocol::serialize(&thrift);
-        let mut context = ChangesetIdContext::new();
-        context.update(&data);
-        let id = context.finish();
-        Blob::new(id, data)
-    }
-
     pub(crate) fn into_thrift(self) -> thrift::BonsaiChangeset {
         thrift::BonsaiChangeset {
             parents: self.inner
@@ -210,6 +192,27 @@ impl BonsaiChangeset {
     }
 }
 
+impl BlobstoreValue for BonsaiChangeset {
+    type Key = ChangesetId;
+
+    fn into_blob(self) -> ChangesetBlob {
+        let thrift = self.into_thrift();
+        let data = compact_protocol::serialize(&thrift);
+        let mut context = ChangesetIdContext::new();
+        context.update(&data);
+        let id = context.finish();
+        Blob::new(id, data)
+    }
+
+    fn from_blob(blob: Blob<Self::Key>) -> Result<Self> {
+        // TODO (T27336549) stop using SyncFailure once thrift is converted to failure
+        let thrift_tc = compact_protocol::deserialize(blob.data().as_ref())
+            .map_err(SyncFailure::new)
+            .context(ErrorKind::BlobDeserializeError("BonsaiChangeset".into()))?;
+        Self::from_thrift(thrift_tc)
+    }
+}
+
 impl Arbitrary for BonsaiChangeset {
     fn arbitrary<G: Gen>(g: &mut G) -> Self {
         // In the future Mononoke would like to support changesets with more parents than 2.
@@ -295,7 +298,7 @@ mod test {
 
         fn blob_roundtrip(cs: BonsaiChangeset) -> bool {
             let blob = cs.clone().into_blob();
-            let cs2 = BonsaiChangeset::from_blob(blob.data().as_ref())
+            let cs2 = BonsaiChangeset::from_blob(blob)
                 .expect("blob roundtrips should always be valid");
             cs == cs2
         }

mononoke-types/src/file_contents.rs

@@ -12,10 +12,10 @@ use quickcheck::{single_shrinker, Arbitrary, Gen};
 
 use rust_thrift::compact_protocol;
 
-use blob::{Blob, ContentBlob};
+use blob::{Blob, BlobstoreValue, ContentBlob};
 use errors::*;
 use thrift;
-use typed_hash::ContentIdContext;
+use typed_hash::{ContentId, ContentIdContext};
 
 /// An enum representing contents for a file. In the future this may have
 /// special support for very large files.
@@ -29,14 +29,6 @@ impl FileContents {
         FileContents::Bytes(b.into())
     }
 
-    pub fn from_blob<T: AsRef<[u8]>>(t: T) -> Result<Self> {
-        // TODO (T27336549) stop using SyncFailure once thrift is converted to failure
-        let thrift_tc = compact_protocol::deserialize(t.as_ref())
-            .map_err(SyncFailure::new)
-            .context(ErrorKind::BlobDeserializeError("FileContents".into()))?;
-        Self::from_thrift(thrift_tc)
-    }
-
     pub(crate) fn from_thrift(fc: thrift::FileContents) -> Result<Self> {
         match fc {
             thrift::FileContents::Bytes(bytes) => Ok(FileContents::Bytes(bytes.into())),
@@ -59,8 +51,18 @@ impl FileContents {
         }
     }
 
-    /// Serialize this structure into a blob.
-    pub fn into_blob(self) -> ContentBlob {
+    pub(crate) fn into_thrift(self) -> thrift::FileContents {
+        match self {
+            // TODO (T26959816) -- allow Thrift to represent binary as Bytes
+            FileContents::Bytes(bytes) => thrift::FileContents::Bytes(bytes.to_vec()),
+        }
+    }
+}
+
+impl BlobstoreValue for FileContents {
+    type Key = ContentId;
+
+    fn into_blob(self) -> ContentBlob {
         let thrift = self.into_thrift();
         let data = compact_protocol::serialize(&thrift);
         let mut context = ContentIdContext::new();
@@ -69,11 +71,12 @@ impl FileContents {
         Blob::new(id, data)
     }
 
-    pub(crate) fn into_thrift(self) -> thrift::FileContents {
-        match self {
-            // TODO (T26959816) -- allow Thrift to represent binary as Bytes
-            FileContents::Bytes(bytes) => thrift::FileContents::Bytes(bytes.to_vec()),
-        }
+    fn from_blob(blob: Blob<Self::Key>) -> Result<Self> {
+        // TODO (T27336549) stop using SyncFailure once thrift is converted to failure
+        let thrift_tc = compact_protocol::deserialize(blob.data().as_ref())
+            .map_err(SyncFailure::new)
+            .context(ErrorKind::BlobDeserializeError("FileContents".into()))?;
+        Self::from_thrift(thrift_tc)
     }
 }
 
@@ -111,7 +114,7 @@ mod test {
 
         fn blob_roundtrip(cs: FileContents) -> bool {
             let blob = cs.clone().into_blob();
-            let cs2 = FileContents::from_blob(blob.data().as_ref())
+            let cs2 = FileContents::from_blob(blob)
                 .expect("blob roundtrips should always be valid");
             cs == cs2
         }

mononoke-types/src/lib.rs

@@ -48,7 +48,7 @@ pub mod hash;
 pub mod path;
 pub mod typed_hash;
 
-pub use blob::{Blob, BlobstoreBytes, ChangesetBlob, ContentBlob};
+pub use blob::{Blob, BlobstoreBytes, BlobstoreValue, ChangesetBlob, ContentBlob};
 pub use bonsai_changeset::BonsaiChangeset;
 pub use datetime::DateTime;
 pub use file_change::{FileChange, FileType};

mononoke-types/src/typed_hash.rs

@@ -10,7 +10,10 @@ use std::str::FromStr;
 use ascii::{AsciiStr, AsciiString};
 use quickcheck::{empty_shrinker, Arbitrary, Gen};
 
+use blob::BlobstoreValue;
+use bonsai_changeset::BonsaiChangeset;
 use errors::*;
+use file_contents::FileContents;
 use hash::{Blake2, Context};
 use thrift;
 
@@ -19,6 +22,9 @@ use thrift;
 
 /// An identifier used throughout Mononoke.
 pub trait MononokeId: Copy + Send + 'static {
+    /// Blobstore value type associated with given MononokeId type
+    type Value: BlobstoreValue<Key = Self>;
+
     /// Return a key suitable for blobstore use.
     fn blobstore_key(&self) -> String;
 
@@ -40,6 +46,7 @@ pub struct ContentId(Blake2);
 macro_rules! impl_typed_hash {
     {
         hash_type => $typed: ident,
+        value_type => $value_type: ident,
         context_type => $typed_context: ident,
         context_key => $key: expr,
     } => {
@@ -134,6 +141,8 @@ macro_rules! impl_typed_hash {
         }
 
         impl MononokeId for $typed {
+            type Value = $value_type;
+
             #[inline]
             fn blobstore_key(&self) -> String {
                 format!(concat!($key, ".blake2.{}"), self.0)
@@ -173,12 +182,14 @@ macro_rules! impl_typed_hash {
 
 impl_typed_hash! {
     hash_type => ChangesetId,
+    value_type => BonsaiChangeset,
     context_type => ChangesetIdContext,
     context_key => "changeset",
 }
 
 impl_typed_hash! {
     hash_type => ContentId,
+    value_type => FileContents,
     context_type => ContentIdContext,
     context_key => "content",
 }