diff --git a/eden/cli/config.py b/eden/cli/config.py index 528acde656..6819713248 100644 --- a/eden/cli/config.py +++ b/eden/cli/config.py @@ -674,6 +674,24 @@ Do you want to run `eden mount %s` instead?""" if foreground: cmd.append("--foreground") + try: + use_mononoke = ( + self.get_config_value("mononoke.use-mononoke").lower() == "true" + ) + except KeyError: + use_mononoke = False + + try: + certificate = self.get_config_value("ssl.client-certificate") + except KeyError: + # probably need to log this case + certificate = None + + if use_mononoke and certificate: + cmd.append("--use_mononoke") + cmd.append("--client_certificate") + cmd.append(certificate) + eden_env = self._build_eden_environment() # Run edenfs using sudo, unless we already have root privileges, diff --git a/eden/fs/store/hg/HgImporter.cpp b/eden/fs/store/hg/HgImporter.cpp index 78cd7e5d04..fb8b23706a 100644 --- a/eden/fs/store/hg/HgImporter.cpp +++ b/eden/fs/store/hg/HgImporter.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -34,6 +35,7 @@ #include "eden/fs/store/hg/HgManifestImporter.h" #include "eden/fs/store/hg/HgProxyHash.h" #include "eden/fs/utils/PathFuncs.h" +#include "eden/fs/utils/SSLContext.h" #include "eden/fs/utils/TimeUtil.h" #if EDEN_HAVE_HG_TREEMANIFEST @@ -89,8 +91,13 @@ DEFINE_int32( 256 * 1024 * 1024, // 256MB "Buffer size for batching LocalStore writes during hg manifest imports"); -namespace { +DEFINE_bool(use_mononoke, false, "Try to fetch trees from Mononoke"); +DEFINE_int32( + mononoke_timeout, + 2000, // msec + "[unit: ms] Timeout for Mononoke requests"); +namespace { using namespace facebook::eden; /** @@ -216,6 +223,7 @@ HgImporter::HgImporter(AbsolutePathPiece repoPath, LocalStore* store) auto options = waitForHelperStart(); initializeTreeManifestImport(options); + initializeMononoke(options); XLOG(DBG1) << "hg_import_helper started for repository " << repoPath_; } @@ -266,6 +274,11 @@ HgImporter::Options HgImporter::waitForHelperStart() { options.treeManifestPackPaths.push_back(cursor.readFixedString(pathLength)); } + if (flags & StartFlag::MONONOKE_SUPPORTED) { + auto nameLength = cursor.readBE(); + options.repoName = cursor.readFixedString(nameLength); + } + return options; } @@ -299,6 +312,39 @@ void HgImporter::initializeTreeManifestImport(const Options& options) { #endif // EDEN_HAVE_HG_TREEMANIFEST } +void HgImporter::initializeMononoke(const Options& options) { +#if EDEN_HAVE_HG_TREEMANIFEST + if (options.repoName.empty()) { + XLOG(DBG2) << "mononoke is disabled because it is not supported."; + return; + } + + if (!FLAGS_use_mononoke) { + XLOG(DBG2) << "mononoke is disabled by command line flags."; + return; + } + + auto executor = folly::getIOExecutor(); + + std::shared_ptr sslContext; + try { + sslContext = buildSSLContext(); + } catch (std::runtime_error& ex) { + XLOG(WARN) << "mononoke is disabled because of build failure when " + "creating SSLContext: " + << ex.what(); + return; + } + + mononoke_ = std::make_unique( + options.repoName, + std::chrono::milliseconds(FLAGS_mononoke_timeout), + executor.get(), + sslContext); + XLOG(DBG2) << "mononoke enabled for repository " << options.repoName; +#endif +} + HgImporter::~HgImporter() { helper_.closeParentFd(STDIN_FILENO); helper_.wait(); @@ -386,6 +432,35 @@ std::unique_ptr HgImporter::importTreeImpl( } } + if (!content.content() && mononoke_) { + // ask Mononoke API Server + XLOG(DBG4) << "importing tree \"" << manifestNode << "\" from mononoke"; + try { + auto mononokeTree = + mononoke_->getTree(manifestNode) + .get(std::chrono::milliseconds(FLAGS_mononoke_timeout)); + std::vector entries; + + for (const auto& entry : mononokeTree->getTreeEntries()) { + auto blobHash = entry.getHash(); + auto entryName = entry.getName(); + auto proxyHash = + HgProxyHash::store(path + entryName, blobHash, writeBatch); + + entries.emplace_back( + proxyHash, entryName.stringPiece(), entry.getType()); + } + + auto tree = std::make_unique(std::move(entries), edenTreeID); + auto serialized = LocalStore::serializeTree(tree.get()); + writeBatch->put( + KeySpace::TreeFamily, edenTreeID, serialized.second.coalesce()); + return tree; + } catch (const std::exception& ex) { + XLOG(WARN) << "got exception from MononokeBackingStore: " << ex.what(); + } + } + if (!content.content()) { // Ask the hg_import_helper script to fetch data for this tree XLOG(DBG1) << "fetching data for tree \"" << path << "\" at manifest node " diff --git a/eden/fs/store/hg/HgImporter.h b/eden/fs/store/hg/HgImporter.h index 8c217a6aee..b89fdfb809 100644 --- a/eden/fs/store/hg/HgImporter.h +++ b/eden/fs/store/hg/HgImporter.h @@ -14,6 +14,7 @@ #include "eden/fs/eden-config.h" #include "eden/fs/store/LocalStore.h" +#include "eden/fs/store/mononoke/MononokeBackingStore.h" #include "eden/fs/utils/PathFuncs.h" namespace folly { @@ -176,6 +177,7 @@ class HgImporter : public Importer { */ enum StartFlag : uint32_t { TREEMANIFEST_SUPPORTED = 0x01, + MONONOKE_SUPPORTED = 0x02, }; /** * Command type values. @@ -212,6 +214,11 @@ class HgImporter : public Importer { * If this vector is empty treemanifest import should not be used. */ std::vector treeManifestPackPaths; + + /** + * The name of the repo + */ + std::string repoName; }; // Forbidden copy constructor and assignment operator @@ -261,6 +268,13 @@ class HgImporter : public Importer { */ void initializeTreeManifestImport(const Options& options); + /** + * Initialize the mononoke_ needed for Mononoke API Server support. + * + * This leaves mononoke_ null if mononoke does not support the repository. + */ + void initializeMononoke(const Options& options); + /** * Send a request to the helper process, asking it to send us the manifest * for the specified revision. @@ -315,6 +329,8 @@ class HgImporter : public Importer { #if EDEN_HAVE_HG_TREEMANIFEST std::vector> dataPackStores_; std::unique_ptr unionStore_; + + std::unique_ptr mononoke_; #endif // EDEN_HAVE_HG_TREEMANIFEST }; } // namespace eden diff --git a/eden/fs/store/hg/hg_import_helper.py b/eden/fs/store/hg/hg_import_helper.py index 73a03c6945..615da9a48a 100755 --- a/eden/fs/store/hg/hg_import_helper.py +++ b/eden/fs/store/hg/hg_import_helper.py @@ -82,6 +82,7 @@ SHA1_NUM_BYTES = 20 PROTOCOL_VERSION = 1 START_FLAGS_TREEMANIFEST_SUPPORTED = 0x01 +START_FLAGS_MONONOKE_SUPPORTED = 0x02 # # Message types. @@ -254,10 +255,13 @@ class HgServer(object): logging.debug("hg_import_helper shutting down normally") return 0 + def _is_mononoke_supported(self, name): + return name in ["fbsource"] + def _gen_options(self): - use_treemanifest = (self.treemanifest is not None) and bool( - getattr(self.repo, "name", None) - ) + repo_name = getattr(self.repo, "name", None) + use_treemanifest = (self.treemanifest is not None) and bool(repo_name) + use_mononoke = use_treemanifest and self._is_mononoke_supported(repo_name) flags = 0 treemanifest_paths = [] @@ -270,6 +274,9 @@ class HgServer(object): shallowutil.getcachepackpath(self.repo, constants.TREEPACK_CATEGORY), ] + if use_mononoke: + flags |= START_FLAGS_MONONOKE_SUPPORTED + # Options format: # - Protocol version number # - Is treemanifest supported? @@ -283,6 +290,10 @@ class HgServer(object): parts.append(struct.pack(b">I", len(path))) parts.append(path) + if use_mononoke: + parts.append(struct.pack(b">I", len(repo_name))) + parts.append(repo_name) + return "".join(parts) def debug(self, msg, *args, **kwargs): diff --git a/eden/fs/utils/SSLContext.cpp b/eden/fs/utils/SSLContext.cpp new file mode 100644 index 0000000000..fb78dfade7 --- /dev/null +++ b/eden/fs/utils/SSLContext.cpp @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2018-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#include "SSLContext.h" + +#include +#include +#include +#include +#include + +DEFINE_string( + client_certificate, + "", + "Path to the client certificate that is used when establishing " + "SSL connection"); + +namespace facebook { +namespace eden { +std::shared_ptr buildSSLContext() { + auto sslContext = std::make_shared(); + if (!FLAGS_client_certificate.empty()) { + XLOG(DBG2) << "build SSLContext with client certificate: " + << FLAGS_client_certificate; + sslContext->loadCertificate(FLAGS_client_certificate.c_str(), "PEM"); + sslContext->loadPrivateKey(FLAGS_client_certificate.c_str(), "PEM"); + } + folly::ssl::SSLCommonOptions::setClientOptions(*sslContext); + + return sslContext; +} +} // namespace eden +} // namespace facebook diff --git a/eden/fs/utils/SSLContext.h b/eden/fs/utils/SSLContext.h new file mode 100644 index 0000000000..e4ee1fe316 --- /dev/null +++ b/eden/fs/utils/SSLContext.h @@ -0,0 +1,21 @@ +/* + * Copyright (c) 2018-present, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + * + */ +#pragma once + +#include + +namespace facebook { +namespace eden { +/** + * Create a folly::SSLcontext with client certificate + */ +std::shared_ptr buildSSLContext(); +} // namespace eden +} // namespace facebook