Integrate with TreeEntry and stores

Summary:
Adding blake3 support into TreeEntry and backing stores.

Note: Http and RE CAS stores don't provide blake3 hash so far. While it would be pretty easy to add support for RE, not sure how hard it would be for the http store.

Reviewed By: chadaustin

Differential Revision: D46268715

fbshipit-source-id: db66e63fe0348eb582a8050f22cdc0ff720ccf85

eden/fs/inodes/VirtualInode.cpp

@@ -189,8 +189,13 @@ EntryAttributes VirtualInode::getEntryAttributesForNonFile(
     objectId = folly::Try<std::optional<ObjectId>>{getObjectId()};
   }
 
+  // TODO: Add blake3 support
   return EntryAttributes{
-      std::move(sha1), std::move(size), std::move(type), std::move(objectId)};
+      std::move(sha1),
+      std::nullopt,
+      std::move(size),
+      std::move(type),
+      std::move(objectId)};
 }
 
 ImmediateFuture<EntryAttributes> VirtualInode::getEntryAttributes(
@@ -281,8 +286,10 @@ ImmediateFuture<EntryAttributes> VirtualInode::getEntryAttributes(
                   folly::Try<std::optional<ObjectId>>{std::move(entryObjectId)};
             }
 
+            // TODO: add blake3 support
             return EntryAttributes{
                 std::move(sha1),
+                std::nullopt,
                 std::move(size),
                 std::move(type),
                 std::move(objectId)};

eden/fs/model/TreeEntry.cpp

@@ -8,6 +8,7 @@
 #include "eden/fs/model/TreeEntry.h"
 
 #include <sys/stat.h>
+#include <cstdint>
 #include <ostream>
 
 #include <folly/Range.h>
@@ -135,7 +136,8 @@ std::ostream& operator<<(std::ostream& os, TreeEntryType type) {
 
 size_t TreeEntry::serializedSize(PathComponentPiece name) const {
   return sizeof(uint8_t) + sizeof(uint16_t) + hash_.size() + sizeof(uint16_t) +
-      name.view().size() + sizeof(uint64_t) + Hash20::RAW_SIZE;
+      name.view().size() + sizeof(uint64_t) + Hash20::RAW_SIZE +
+      sizeof(uint8_t) + Hash32::RAW_SIZE;
 }
 
 void TreeEntry::serialize(PathComponentPiece name, Appender& appender) const {
@@ -158,6 +160,13 @@ void TreeEntry::serialize(PathComponentPiece name, Appender& appender) const {
   } else {
     appender.push(kZeroHash.getBytes());
   }
+
+  // we need to be backward compatible with the old serialization format
+  // so adding a byte (with flipped bits) to distinguish between a possible
+  // blake3 hash and the next entry type because we have access to the entire
+  // serialized tree
+  appender.write(0xff, sizeof(uint8_t));
+  appender.push(contentBlake3_.value_or(kZeroHash32).getBytes());
 }
 
 std::optional<std::pair<PathComponent, TreeEntry>> TreeEntry::deserialize(
@@ -238,8 +247,24 @@ std::optional<std::pair<PathComponent, TreeEntry>> TreeEntry::deserialize(
     sha1 = sha1_raw;
   }
 
+  std::optional<Hash32> blake3;
+  if (!data.empty() && static_cast<uint8_t>(data.data()[0]) == 0xff) {
+    data.advance(1);
+
+    if (data.size() >= Hash32::RAW_SIZE) {
+      blake3.emplace();
+      auto blake3Bytes = blake3->mutableBytes();
+      memcpy(blake3Bytes.data(), data.data(), Hash32::RAW_SIZE);
+      data.advance(Hash32::RAW_SIZE);
+      if (*blake3 == kZeroHash32) {
+        blake3.reset();
+      }
+    }
+  }
+
   return std::pair{
-      std::move(name), TreeEntry{hash, (TreeEntryType)type, size, sha1}};
+      std::move(name),
+      TreeEntry{hash, (TreeEntryType)type, size, sha1, blake3}};
 }
 
 } // namespace facebook::eden

eden/fs/model/TreeEntry.h

@@ -42,6 +42,7 @@ struct EntryAttributes {
   // regular files, executable files, and symlinks. FIFOs or sockets for
   // example would fall into the nullopt case.
   std::optional<folly::Try<Hash20>> sha1;
+  std::optional<folly::Try<Hash32>> blake3;
   std::optional<folly::Try<uint64_t>> size;
   std::optional<folly::Try<std::optional<TreeEntryType>>> type;
   std::optional<folly::Try<std::optional<ObjectId>>> objectId;
@@ -77,11 +78,13 @@ class TreeEntry {
       const ObjectId& hash,
       TreeEntryType type,
       std::optional<uint64_t> size,
-      std::optional<Hash20> contentSha1)
+      std::optional<Hash20> contentSha1,
+      std::optional<Hash32> contentBlake3)
       : type_(type),
         hash_(std::move(hash)),
         size_(size),
-        contentSha1_(contentSha1) {}
+        contentSha1_(contentSha1),
+        contentBlake3_(contentBlake3) {}
 
   const ObjectId& getObjectId() const {
     return hash_;
@@ -142,6 +145,10 @@ class TreeEntry {
     return contentSha1_;
   }
 
+  const std::optional<Hash32>& getContentBlake3() const {
+    return contentBlake3_;
+  }
+
   /**
    * Computes exact serialized size of this entry.
    */
@@ -163,6 +170,7 @@ class TreeEntry {
   ObjectId hash_;
   std::optional<uint64_t> size_;
   std::optional<Hash20> contentSha1_;
+  std::optional<Hash32> contentBlake3_;
 
   static constexpr uint64_t NO_SIZE = std::numeric_limits<uint64_t>::max();
 };

eden/fs/model/test/TreeEntryTest.cpp

@@ -6,6 +6,7 @@
  */
 
 #include <folly/portability/GTest.h>
+#include <optional>
 
 #include "eden/fs/model/TreeEntry.h"
 #include "eden/fs/testharness/TestUtil.h"
@@ -64,22 +65,32 @@ TEST(TreeEntry, testEntrySize) {
 
 TEST(TreeEntry, testEntryAttributesEqual) {
   EntryAttributes nullAttributes{
-      std::nullopt, std::nullopt, std::nullopt, std::nullopt};
+      std::nullopt, std::nullopt, std::nullopt, std::nullopt, std::nullopt};
   EntryAttributes error1Attributes{
+      std::nullopt,
       std::nullopt,
       folly::Try<uint64_t>{newEdenError(std::exception{})},
       std::nullopt,
       std::nullopt};
   EntryAttributes error2Attributes{
+      std::nullopt,
       std::nullopt,
       folly::Try<uint64_t>{
           newEdenError(std::runtime_error{"some other error"})},
       std::nullopt,
       std::nullopt};
   EntryAttributes real1Attributes{
-      std::nullopt, folly::Try<uint64_t>{1}, std::nullopt, std::nullopt};
+      std::nullopt,
+      std::nullopt,
+      folly::Try<uint64_t>{1},
+      std::nullopt,
+      std::nullopt};
   EntryAttributes real2Attributes{
-      std::nullopt, folly::Try<uint64_t>{2}, std::nullopt, std::nullopt};
+      std::nullopt,
+      std::nullopt,
+      folly::Try<uint64_t>{2},
+      std::nullopt,
+      std::nullopt};
 
   EXPECT_EQ(nullAttributes, nullAttributes);
   EXPECT_NE(nullAttributes, error1Attributes);

eden/fs/store/hg/HgDatapackStore.cpp

@@ -48,13 +48,18 @@ Tree::value_type fromRawTreeEntry(
     HgObjectIdFormat hgObjectIdFormat) {
   std::optional<uint64_t> size;
   std::optional<Hash20> contentSha1;
+  std::optional<Hash32> contentBlake3;
 
   if (entry.size != nullptr) {
     size = *entry.size;
   }
 
   if (entry.content_sha1 != nullptr) {
-    contentSha1 = Hash20{*entry.content_sha1};
+    contentSha1.emplace(*entry.content_sha1);
+  }
+
+  if (entry.content_blake3 != nullptr) {
+    contentBlake3.emplace(*entry.content_blake3);
   }
 
   auto name = PathComponent(folly::StringPiece{entry.name.asByteRange()});
@@ -64,7 +69,11 @@ Tree::value_type fromRawTreeEntry(
   auto proxyHash = HgProxyHash::store(fullPath, hash, hgObjectIdFormat);
 
   auto treeEntry = TreeEntry{
-      proxyHash, fromRawTreeEntryType(entry.ttype), size, contentSha1};
+      proxyHash,
+      fromRawTreeEntryType(entry.ttype),
+      size,
+      contentSha1,
+      contentBlake3};
   return {std::move(name), std::move(treeEntry)};
 }
 
@@ -271,8 +280,12 @@ BlobMetadataPtr HgDatapackStore::getLocalBlobMetadata(
   auto metadata =
       store_.getBlobMetadata(hgInfo.byteHash(), true /*local_only*/);
   if (metadata) {
-    return std::make_shared<BlobMetadataPtr::element_type>(
-        BlobMetadata{Hash20{metadata->content_sha1}, metadata->total_size});
+    std::optional<Hash32> blake3;
+    if (metadata->content_blake3 != nullptr) {
+      blake3.emplace(*metadata->content_blake3);
+    }
+    return std::make_shared<BlobMetadataPtr::element_type>(BlobMetadata{
+        Hash20{metadata->content_sha1}, blake3, metadata->total_size});
   }
   return nullptr;
 }
@@ -317,8 +330,13 @@ void HgDatapackStore::getBlobMetadataBatch(
               }
 
               auto& aux = auxTry.value();
+              std::optional<Hash32> blake3;
+              if (aux->content_blake3 != nullptr) {
+                blake3.emplace(*aux->content_blake3);
+              }
+
               return folly::Try{std::make_shared<BlobMetadataPtr::element_type>(
-                  Hash20{aux->content_sha1}, aux->total_size)};
+                  Hash20{aux->content_sha1}, blake3, aux->total_size)};
             });
 
         // Make sure that we're stopping this watch.

eden/fs/store/test/TreeTest.cpp

@@ -25,10 +25,16 @@ TEST_P(LocalStoreTest, testReadAndWriteTree) {
 
   StringPiece childContents("blah\n");
   auto childSha1 = Hash20::sha1(folly::ByteRange{childContents});
+  auto childBlake3 = Hash32::blake3(folly::ByteRange{childContents});
   auto size = childContents.size();
   Tree::container entries{kPathMapDefaultCaseSensitive};
   entries.emplace(
-      "entry1"_pc, childHash1, TreeEntryType::REGULAR_FILE, size, childSha1);
+      "entry1"_pc,
+      childHash1,
+      TreeEntryType::REGULAR_FILE,
+      size,
+      childSha1,
+      childBlake3);
   entries.emplace("entry2"_pc, childHash2, TreeEntryType::REGULAR_FILE);
   auto tree = Tree{std::move(entries), hash};