mononoke: apiserver: augment /tree method with size and content sha1 fields

Summary:
This augments `/tree` to yields the size and content sha1 hash for the entries.
This is important for Eden and avoids additional round trips to the server.

The content hashing is the portion that I expect some push back on,
because it doesn't appear to be cached today and the implementation
here does a simplistic fetch and hash.  By doing this we hope to
squash out a potential later fetch of the entire contents when
buck build is run.

Differential Revision: D10865588

fbshipit-source-id: c020ef07b99d8a5e8b2f8f7b699bf15e750d60a5

apiserver/src/actor/model.rs

@@ -10,10 +10,14 @@ use std::convert::TryFrom;
 use std::str;
 
 use chrono::{DateTime, FixedOffset};
-use failure::Error;
+use failure::{err_msg, Error};
 
 use blobrepo::HgBlobChangeset;
+use futures::prelude::*;
+use futures_ext::{spawn_future, BoxFuture, FutureExt};
 use mercurial_types::{Changeset as HgChangeset, Entry as HgEntry, Type};
+use mercurial_types::hash::Sha1;
+use mercurial_types::manifest::Content;
 
 #[derive(Serialize)]
 pub enum FileType {
@@ -62,6 +66,55 @@ impl TryFrom<Box<HgEntry + Sync>> for Entry {
     }
 }
 
+#[derive(Serialize)]
+pub struct EntryWithSizeAndContentHash {
+    name: String,
+    #[serde(rename = "type")]
+    ttype: FileType,
+    hash: String,
+    size: Option<usize>,
+    content_sha1: Option<String>,
+}
+
+impl EntryWithSizeAndContentHash {
+    pub fn materialize_future(entry: Box<HgEntry + Sync>) -> BoxFuture<Self, Error> {
+        let name = try_boxfuture!(
+            entry
+                .get_name()
+                .map(|name| name.to_bytes())
+                .ok_or_else(|| err_msg("HgEntry has no name!?"))
+        );
+        // FIXME: json cannot represent non-UTF8 file names
+        let name = try_boxfuture!(String::from_utf8(name));
+        let ttype = entry.get_type().into();
+        let hash = entry.get_hash().to_string();
+
+        spawn_future(entry.get_content().and_then(move |content| {
+            let size = match &content {
+                Content::File(contents)
+                | Content::Executable(contents)
+                | Content::Symlink(contents) => Some(contents.size()),
+                Content::Tree(manifest) => Some(manifest.list().count()),
+            };
+            Ok(EntryWithSizeAndContentHash {
+                name,
+                ttype,
+                hash,
+                size,
+                content_sha1: match content {
+                    Content::File(contents)
+                    | Content::Executable(contents)
+                    | Content::Symlink(contents) => {
+                        let sha1 = Sha1::from(contents.as_bytes().as_ref());
+                        Some(sha1.to_hex().to_string())
+                    }
+                    Content::Tree(_) => None,
+                },
+            })
+        })).boxify()
+    }
+}
+
 #[derive(Serialize)]
 pub struct Changeset {
     manifest: String,

apiserver/src/actor/repo.rs

@@ -10,6 +10,7 @@ use std::sync::Arc;
 use bytes::Bytes;
 use failure::{err_msg, Error};
 use futures::{Future, IntoFuture};
+use futures::future::join_all;
 use futures::sync::oneshot;
 use futures_ext::{BoxFuture, FutureExt};
 use slog::Logger;
@@ -31,7 +32,7 @@ use from_string as FS;
 
 use super::{MononokeRepoQuery, MononokeRepoResponse};
 use super::lfs::{build_response, BatchRequest};
-use super::model::Entry;
+use super::model::{Entry, EntryWithSizeAndContentHash};
 
 pub struct MononokeRepo {
     repo: Arc<BlobRepo>,
@@ -201,12 +202,13 @@ impl MononokeRepo {
         self.repo
             .get_manifest_by_nodeid(&treemanifestid)
             .map(|tree| {
-                tree.list()
-                    .filter_map(|entry| -> Option<Entry> { entry.try_into().ok() })
-            })
-            .map(|files| MononokeRepoResponse::GetTree {
-                files: Box::new(files),
+                join_all(
+                    tree.list()
+                        .map(|entry| EntryWithSizeAndContentHash::materialize_future(entry)),
+                )
             })
+            .flatten()
+            .map(|files| MononokeRepoResponse::GetTree { files })
             .from_err()
             .boxify()
     }

apiserver/src/actor/response.rs

@@ -11,7 +11,7 @@ use actix_web::{Body, HttpRequest, HttpResponse, Json, Responder};
 use bytes::Bytes;
 
 use super::lfs::BatchResponse;
-use super::model::{Changeset, Entry};
+use super::model::{Changeset, Entry, EntryWithSizeAndContentHash};
 
 pub enum MononokeRepoResponse {
     GetRawFile {
@@ -24,7 +24,7 @@ pub enum MononokeRepoResponse {
         files: Box<Iterator<Item = Entry> + Send>,
     },
     GetTree {
-        files: Box<Iterator<Item = Entry> + Send>,
+        files: Vec<EntryWithSizeAndContentHash>,
     },
     GetChangeset {
         changeset: Changeset,
@@ -56,9 +56,12 @@ impl Responder for MononokeRepoResponse {
 
         match self {
             GetRawFile { content } | GetBlobContent { content } => Ok(binary_response(content)),
-            ListDirectory { files } | GetTree { files } => {
+            ListDirectory { files } => {
                 Json(files.collect::<Vec<_>>()).respond_to(req)
             }
+            GetTree{files}=> {
+                Json(files).respond_to(req)
+            }
             GetChangeset { changeset } => Json(changeset).respond_to(req),
             IsAncestor { answer } => Ok(binary_response({
                 if answer {

tests/integration/test-apiserver.t

@@ -14,7 +14,7 @@ setup testing repo for mononoke
   $ SHA=$(sha256sum test | awk '{print $1;}')
   $ ln -s test link
   $ mkdir -p folder/subfolder
-  $ touch folder/subfolder/.keep
+  $ echo "hello" > folder/subfolder/.keep
   $ hg add test link folder/subfolder/.keep
   $ hg commit -ma
   $ COMMIT1=$(hg --debug id -i)
@@ -184,7 +184,7 @@ test folder list
       {
           "name": "subfolder",
           "type": "tree",
-          "hash": "9b5497965e634f261cca0247a7a48b709a7be2b9"
+          "hash": "732eacf2be3265bd6bc4d2c205434b280f446cbf"
       }
   ]
 
@@ -195,7 +195,7 @@ test folder list
       {
           "name": ".keep",
           "type": "file",
-          "hash": "b80de5d138758541c5f05265ad144ab9fa86d1db"
+          "hash": "2c186c8c5bc0df5af5b951afe407d803f9e6b8c9"
       }
   ]
 
@@ -214,7 +214,7 @@ test get blob by hash
   $ diff output - <<< $TEST_CONTENT
 
   $ sslcurl -w "\n%{http_code}" $APISERVER/repo/blob/$TREEHASH | extract_json_error
-  9b5497965e634f261cca0247a7a48b709a7be2b9 is not found
+  732eacf2be3265bd6bc4d2c205434b280f446cbf is not found
   404
 
   $ sslcurl -w "\n%{http_code}" $APISERVER/repo/blob/0000 | extract_json_error
@@ -231,7 +231,9 @@ test get tree
       {
           "name": ".keep",
           "type": "file",
-          "hash": "b80de5d138758541c5f05265ad144ab9fa86d1db"
+          "hash": "2c186c8c5bc0df5af5b951afe407d803f9e6b8c9",
+          "size": 6,
+          "content_sha1": "f572d396fae9206628714fb2ce00f72e94f2258f"
       }
   ]