mononoke: same storage keys for manifest entries as for the file entries

Summary:
A preparation step before blob importing of tree manifest repos to blobrepo.

`get_parents()` method of BlobEntry reads parents from the blobstore. It works fine for file entries because file entries can stores its parents in the blobstore. With tree manifests BlobEntry can contain also tree manifest entries, and that means that tree manifest entries parents should also be stored somewhere in the blobstore.

I suggest to use the same logic for the tree manifest entries as for the file entries. File and manifest entries have two blobstore entries - one stores hash of the content and parents, another stores the actual content.

To do this I moved `RawNodeBlob` and `get_node()` to the separate module and made fields public.

Reviewed By: jsgf

Differential Revision: D5622342

fbshipit-source-id: c9f0c446107d4697b042544ff8b37a159064f061

blobrepo/src/file.rs

@@ -6,11 +6,9 @@
 
 //! Plain files, symlinks
 
-use futures::future::{BoxFuture, Future, IntoFuture};
+use futures::future::{BoxFuture, Future};
 
-use bincode;
-
-use mercurial_types::{Blob, NodeHash, Parents, Path, hash};
+use mercurial_types::{Blob, NodeHash, Parents, Path};
 use mercurial_types::manifest::{Content, Entry, Manifest, Type};
 
 use blobstore::Blobstore;
@@ -19,6 +17,8 @@ use errors::*;
 
 use manifest::BlobManifest;
 
+use utils::{get_node, RawNodeBlob};
+
 pub struct BlobEntry<B> {
     blobstore: B,
     path: Path, // XXX full path? Parent reference?
@@ -26,29 +26,6 @@ pub struct BlobEntry<B> {
     ty: Type,
 }
 
-#[derive(Debug, Copy, Clone)]
-#[derive(Serialize, Deserialize)]
-pub struct RawNodeBlob {
-    parents: Parents,
-    blob: hash::Sha1,
-}
-
-fn get_node<B>(blobstore: &B,  nodeid: NodeHash) -> BoxFuture<RawNodeBlob, Error>
-where B: Blobstore<Key = String>,
-      B::ValueOut: AsRef<[u8]>,
-{
-    let key = format!("node:{}.bincode", nodeid);
-
-    blobstore
-        .get(&key)
-        .map_err(blobstore_err)
-        .and_then(move |got| got.ok_or(ErrorKind::NodeMissing(nodeid).into()))
-        .and_then(move |blob| {
-            bincode::deserialize(blob.as_ref()).into_future().from_err()
-        })
-        .boxed()
-}
-
 pub fn fetch_file_blob_from_blobstore<B>(
     blobstore: B,
     nodeid: NodeHash,

blobrepo/src/lib.rs

@@ -31,6 +31,7 @@ mod changeset;
 mod manifest;
 mod file;
 mod errors;
+mod utils;
 
 pub use errors::*;
 

blobrepo/src/manifest.rs

@@ -18,6 +18,7 @@ use blobstore::Blobstore;
 
 use errors::*;
 use file::BlobEntry;
+use utils::get_node;
 
 pub struct BlobManifest<B> {
     blobstore: B,
@@ -30,11 +31,14 @@ where
     B::ValueOut: AsRef<[u8]>,
 {
     pub fn load(blobstore: &B, manifestid: &NodeHash) -> BoxFuture<Option<Self>, Error> {
-        let key = format!("manifest:{}", manifestid);
-
-        blobstore
-            .get(&key)
-            .map_err(blobstore_err)
+        get_node(blobstore, manifestid.clone())
+            .and_then({
+                let blobstore = blobstore.clone();
+                move |nodeblob| {
+                    let blobkey = format!("sha1:{}", nodeblob.blob);
+                    blobstore.get(&blobkey).map_err(blobstore_err)
+                }
+            })
             .and_then({
                 let blobstore = blobstore.clone();
                 move |got| match got {

blobrepo/src/utils.rs

@@ -0,0 +1,37 @@
+// Copyright (c) 2004-present, Facebook, Inc.
+// All Rights Reserved.
+//
+// This software may be used and distributed according to the terms of the
+// GNU General Public License version 2 or any later version.
+
+use futures::future::{BoxFuture, Future, IntoFuture};
+
+use bincode;
+
+use blobstore::Blobstore;
+use mercurial_types::{NodeHash, Parents, hash};
+
+use errors::*;
+
+#[derive(Debug, Copy, Clone)]
+#[derive(Serialize, Deserialize)]
+pub struct RawNodeBlob {
+    pub parents: Parents,
+    pub blob: hash::Sha1,
+}
+
+pub fn get_node<B>(blobstore: &B,  nodeid: NodeHash) -> BoxFuture<RawNodeBlob, Error>
+where B: Blobstore<Key = String>,
+      B::ValueOut: AsRef<[u8]>,
+{
+    let key = format!("node:{}.bincode", nodeid);
+
+    blobstore
+        .get(&key)
+        .map_err(blobstore_err)
+        .and_then(move |got| got.ok_or(ErrorKind::NodeMissing(nodeid).into()))
+        .and_then(move |blob| {
+            bincode::deserialize(blob.as_ref()).into_future().from_err()
+        })
+        .boxed()
+}

cmds/blobimport.rs

@@ -192,13 +192,23 @@ fn copy_changeset(
                     .get_manifest_blob_by_nodeid(&mfid)
                     .from_err()
                     .and_then(move |blob| {
-                        let putmf = blobstore
-                            .put(
-                                format!("manifest:{}", mfid),
-                                blob.as_blob().as_slice().expect("missing blob").into(),
-                            )
-                            .map_err(Into::into);
+                        let bytes = Bytes::from(blob.as_blob().as_slice().unwrap());
+                        let nodeblob = NodeBlob {
+                            parents: blob.parents().clone(),
+                            blob: hash::Sha1::from(bytes.as_ref()),
+                        };
+                        let nodekey = format!("node:{}.bincode", mfid);
+                        let blobkey = format!("sha1:{}", nodeblob.blob);
+                        let nodeblob = bincode::serialize(&nodeblob, bincode::Bounded(4096))
+                           .expect("bincode serialize failed");
 
+                        // TODO: blobstore.putv?
+                        let node = blobstore
+                            .put(nodekey, Bytes::from(nodeblob))
+                            .map_err(Into::into);
+                        let putblob = blobstore.put(blobkey, bytes).map_err(Into::into);
+
+                        let putmf = putblob.join(node);
                         // Get the listing of entries and fetch each of those
                         let files = RevlogManifest::new(revlog.clone(), blob)
                             .map_err(|err| {