Use MononokeBackingStore when fetching trees

Summary:
This commit integrates Mononoke API Server with eden:

* Added two new command line options: `--client_certificate` and `--use_mononoke`.
* Added two new config values: `ssl.client-certificate` and `mononoke.use-mononoke`.
* Made `HgImporter` return the repo name along with treemanifest options.
* Make `HgImporter` ask Mononoke API Server for tree data when the desired tree is not present in local Mercurial store.

Reviewed By: chadaustin

Differential Revision: D9183035

fbshipit-source-id: e328fb3237d10c545c8af71f856007ad6c487061

eden/cli/config.py

@@ -674,6 +674,24 @@ Do you want to run `eden mount %s` instead?"""
         if foreground:
             cmd.append("--foreground")
 
+        try:
+            use_mononoke = (
+                self.get_config_value("mononoke.use-mononoke").lower() == "true"
+            )
+        except KeyError:
+            use_mononoke = False
+
+        try:
+            certificate = self.get_config_value("ssl.client-certificate")
+        except KeyError:
+            # probably need to log this case
+            certificate = None
+
+        if use_mononoke and certificate:
+            cmd.append("--use_mononoke")
+            cmd.append("--client_certificate")
+            cmd.append(certificate)
+
         eden_env = self._build_eden_environment()
 
         # Run edenfs using sudo, unless we already have root privileges,

eden/fs/store/hg/HgImporter.cpp

@@ -15,6 +15,7 @@
 #include <folly/FileUtil.h>
 #include <folly/container/Array.h>
 #include <folly/dynamic.h>
+#include <folly/executors/GlobalExecutor.h>
 #include <folly/experimental/EnvUtil.h>
 #include <folly/futures/Future.h>
 #include <folly/io/Cursor.h>
@@ -34,6 +35,7 @@
 #include "eden/fs/store/hg/HgManifestImporter.h"
 #include "eden/fs/store/hg/HgProxyHash.h"
 #include "eden/fs/utils/PathFuncs.h"
+#include "eden/fs/utils/SSLContext.h"
 #include "eden/fs/utils/TimeUtil.h"
 
 #if EDEN_HAVE_HG_TREEMANIFEST
@@ -89,8 +91,13 @@ DEFINE_int32(
     256 * 1024 * 1024, // 256MB
     "Buffer size for batching LocalStore writes during hg manifest imports");
 
-namespace {
+DEFINE_bool(use_mononoke, false, "Try to fetch trees from Mononoke");
+DEFINE_int32(
+    mononoke_timeout,
+    2000, // msec
+    "[unit: ms] Timeout for Mononoke requests");
 
+namespace {
 using namespace facebook::eden;
 
 /**
@@ -216,6 +223,7 @@ HgImporter::HgImporter(AbsolutePathPiece repoPath, LocalStore* store)
 
   auto options = waitForHelperStart();
   initializeTreeManifestImport(options);
+  initializeMononoke(options);
   XLOG(DBG1) << "hg_import_helper started for repository " << repoPath_;
 }
 
@@ -266,6 +274,11 @@ HgImporter::Options HgImporter::waitForHelperStart() {
     options.treeManifestPackPaths.push_back(cursor.readFixedString(pathLength));
   }
 
+  if (flags & StartFlag::MONONOKE_SUPPORTED) {
+    auto nameLength = cursor.readBE<uint32_t>();
+    options.repoName = cursor.readFixedString(nameLength);
+  }
+
   return options;
 }
 
@@ -299,6 +312,39 @@ void HgImporter::initializeTreeManifestImport(const Options& options) {
 #endif // EDEN_HAVE_HG_TREEMANIFEST
 }
 
+void HgImporter::initializeMononoke(const Options& options) {
+#if EDEN_HAVE_HG_TREEMANIFEST
+  if (options.repoName.empty()) {
+    XLOG(DBG2) << "mononoke is disabled because it is not supported.";
+    return;
+  }
+
+  if (!FLAGS_use_mononoke) {
+    XLOG(DBG2) << "mononoke is disabled by command line flags.";
+    return;
+  }
+
+  auto executor = folly::getIOExecutor();
+
+  std::shared_ptr<folly::SSLContext> sslContext;
+  try {
+    sslContext = buildSSLContext();
+  } catch (std::runtime_error& ex) {
+    XLOG(WARN) << "mononoke is disabled because of build failure when "
+                  "creating SSLContext: "
+               << ex.what();
+    return;
+  }
+
+  mononoke_ = std::make_unique<MononokeBackingStore>(
+      options.repoName,
+      std::chrono::milliseconds(FLAGS_mononoke_timeout),
+      executor.get(),
+      sslContext);
+  XLOG(DBG2) << "mononoke enabled for repository " << options.repoName;
+#endif
+}
+
 HgImporter::~HgImporter() {
   helper_.closeParentFd(STDIN_FILENO);
   helper_.wait();
@@ -386,6 +432,35 @@ std::unique_ptr<Tree> HgImporter::importTreeImpl(
     }
   }
 
+  if (!content.content() && mononoke_) {
+    // ask Mononoke API Server
+    XLOG(DBG4) << "importing tree \"" << manifestNode << "\" from mononoke";
+    try {
+      auto mononokeTree =
+          mononoke_->getTree(manifestNode)
+              .get(std::chrono::milliseconds(FLAGS_mononoke_timeout));
+      std::vector<TreeEntry> entries;
+
+      for (const auto& entry : mononokeTree->getTreeEntries()) {
+        auto blobHash = entry.getHash();
+        auto entryName = entry.getName();
+        auto proxyHash =
+            HgProxyHash::store(path + entryName, blobHash, writeBatch);
+
+        entries.emplace_back(
+            proxyHash, entryName.stringPiece(), entry.getType());
+      }
+
+      auto tree = std::make_unique<Tree>(std::move(entries), edenTreeID);
+      auto serialized = LocalStore::serializeTree(tree.get());
+      writeBatch->put(
+          KeySpace::TreeFamily, edenTreeID, serialized.second.coalesce());
+      return tree;
+    } catch (const std::exception& ex) {
+      XLOG(WARN) << "got exception from MononokeBackingStore: " << ex.what();
+    }
+  }
+
   if (!content.content()) {
     // Ask the hg_import_helper script to fetch data for this tree
     XLOG(DBG1) << "fetching data for tree \"" << path << "\" at manifest node "

eden/fs/store/hg/HgImporter.h

@@ -14,6 +14,7 @@
 
 #include "eden/fs/eden-config.h"
 #include "eden/fs/store/LocalStore.h"
+#include "eden/fs/store/mononoke/MononokeBackingStore.h"
 #include "eden/fs/utils/PathFuncs.h"
 
 namespace folly {
@@ -176,6 +177,7 @@ class HgImporter : public Importer {
    */
   enum StartFlag : uint32_t {
     TREEMANIFEST_SUPPORTED = 0x01,
+    MONONOKE_SUPPORTED = 0x02,
   };
   /**
    * Command type values.
@@ -212,6 +214,11 @@ class HgImporter : public Importer {
      * If this vector is empty treemanifest import should not be used.
      */
     std::vector<std::string> treeManifestPackPaths;
+
+    /**
+     * The name of the repo
+     */
+    std::string repoName;
   };
 
   // Forbidden copy constructor and assignment operator
@@ -261,6 +268,13 @@ class HgImporter : public Importer {
    */
   void initializeTreeManifestImport(const Options& options);
 
+  /**
+   * Initialize the mononoke_ needed for Mononoke API Server support.
+   *
+   * This leaves mononoke_ null if mononoke does not support the repository.
+   */
+  void initializeMononoke(const Options& options);
+
   /**
    * Send a request to the helper process, asking it to send us the manifest
    * for the specified revision.
@@ -315,6 +329,8 @@ class HgImporter : public Importer {
 #if EDEN_HAVE_HG_TREEMANIFEST
   std::vector<std::unique_ptr<DatapackStore>> dataPackStores_;
   std::unique_ptr<UnionDatapackStore> unionStore_;
+
+  std::unique_ptr<MononokeBackingStore> mononoke_;
 #endif // EDEN_HAVE_HG_TREEMANIFEST
 };
 } // namespace eden

eden/fs/store/hg/hg_import_helper.py

@@ -82,6 +82,7 @@ SHA1_NUM_BYTES = 20
 PROTOCOL_VERSION = 1
 
 START_FLAGS_TREEMANIFEST_SUPPORTED = 0x01
+START_FLAGS_MONONOKE_SUPPORTED = 0x02
 
 #
 # Message types.
@@ -254,10 +255,13 @@ class HgServer(object):
         logging.debug("hg_import_helper shutting down normally")
         return 0
 
+    def _is_mononoke_supported(self, name):
+        return name in ["fbsource"]
+
     def _gen_options(self):
-        use_treemanifest = (self.treemanifest is not None) and bool(
-            getattr(self.repo, "name", None)
-        )
+        repo_name = getattr(self.repo, "name", None)
+        use_treemanifest = (self.treemanifest is not None) and bool(repo_name)
+        use_mononoke = use_treemanifest and self._is_mononoke_supported(repo_name)
 
         flags = 0
         treemanifest_paths = []
@@ -270,6 +274,9 @@ class HgServer(object):
                 shallowutil.getcachepackpath(self.repo, constants.TREEPACK_CATEGORY),
             ]
 
+        if use_mononoke:
+            flags |= START_FLAGS_MONONOKE_SUPPORTED
+
         # Options format:
         # - Protocol version number
         # - Is treemanifest supported?
@@ -283,6 +290,10 @@ class HgServer(object):
             parts.append(struct.pack(b">I", len(path)))
             parts.append(path)
 
+        if use_mononoke:
+            parts.append(struct.pack(b">I", len(repo_name)))
+            parts.append(repo_name)
+
         return "".join(parts)
 
     def debug(self, msg, *args, **kwargs):

eden/fs/utils/SSLContext.cpp

@@ -0,0 +1,39 @@
+/*
+ *  Copyright (c) 2018-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under the BSD-style license found in the
+ *  LICENSE file in the root directory of this source tree. An additional grant
+ *  of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+#include "SSLContext.h"
+
+#include <folly/io/async/SSLContext.h>
+#include <folly/io/async/SSLOptions.h>
+#include <folly/logging/xlog.h>
+#include <gflags/gflags.h>
+#include <glog/logging.h>
+
+DEFINE_string(
+    client_certificate,
+    "",
+    "Path to the client certificate that is used when establishing "
+    "SSL connection");
+
+namespace facebook {
+namespace eden {
+std::shared_ptr<folly::SSLContext> buildSSLContext() {
+  auto sslContext = std::make_shared<folly::SSLContext>();
+  if (!FLAGS_client_certificate.empty()) {
+    XLOG(DBG2) << "build SSLContext with client certificate: "
+               << FLAGS_client_certificate;
+    sslContext->loadCertificate(FLAGS_client_certificate.c_str(), "PEM");
+    sslContext->loadPrivateKey(FLAGS_client_certificate.c_str(), "PEM");
+  }
+  folly::ssl::SSLCommonOptions::setClientOptions(*sslContext);
+
+  return sslContext;
+}
+} // namespace eden
+} // namespace facebook

eden/fs/utils/SSLContext.h

@@ -0,0 +1,21 @@
+/*
+ *  Copyright (c) 2018-present, Facebook, Inc.
+ *  All rights reserved.
+ *
+ *  This source code is licensed under the BSD-style license found in the
+ *  LICENSE file in the root directory of this source tree. An additional grant
+ *  of patent rights can be found in the PATENTS file in the same directory.
+ *
+ */
+#pragma once
+
+#include <folly/io/async/SSLContext.h>
+
+namespace facebook {
+namespace eden {
+/**
+ * Create a folly::SSLcontext with client certificate
+ */
+std::shared_ptr<folly::SSLContext> buildSSLContext();
+} // namespace eden
+} // namespace facebook