2016-06-14 01:15:32 +03:00
|
|
|
/*
|
2019-06-20 02:58:25 +03:00
|
|
|
* Copyright (c) Facebook, Inc. and its affiliates.
|
2016-06-14 01:15:32 +03:00
|
|
|
*
|
2019-06-20 02:58:25 +03:00
|
|
|
* This software may be used and distributed according to the terms of the
|
|
|
|
* GNU General Public License version 2.
|
2016-06-14 01:15:32 +03:00
|
|
|
*/
|
2019-10-11 15:26:59 +03:00
|
|
|
|
2016-06-14 01:15:32 +03:00
|
|
|
#pragma once
|
|
|
|
|
2019-03-12 00:20:44 +03:00
|
|
|
#include "eden/fs/config/EdenConfig.h"
|
2018-09-25 21:08:41 +03:00
|
|
|
#include "eden/fs/eden-config.h"
|
2016-06-14 01:15:32 +03:00
|
|
|
#include "eden/fs/store/BackingStore.h"
|
2018-09-25 21:08:41 +03:00
|
|
|
#include "eden/fs/store/LocalStore.h"
|
2019-10-15 23:37:04 +03:00
|
|
|
#include "eden/fs/telemetry/EdenStats.h"
|
2019-10-01 23:11:16 +03:00
|
|
|
#include "eden/fs/utils/PathFuncs.h"
|
2016-06-14 01:15:32 +03:00
|
|
|
|
2017-11-15 20:46:11 +03:00
|
|
|
#include <folly/Executor.h>
|
2016-06-14 01:15:32 +03:00
|
|
|
#include <folly/Range.h>
|
|
|
|
#include <folly/Synchronized.h>
|
2019-04-11 05:55:31 +03:00
|
|
|
#include <memory>
|
2018-10-24 03:03:12 +03:00
|
|
|
#include <optional>
|
2016-06-14 01:15:32 +03:00
|
|
|
|
2019-10-09 23:00:50 +03:00
|
|
|
#ifdef EDEN_HAVE_RUST_DATAPACK
|
2019-10-01 23:11:16 +03:00
|
|
|
#include "eden/fs/store/hg/HgDatapackStore.h" // @manual
|
2019-08-28 03:02:49 +03:00
|
|
|
#endif
|
|
|
|
|
2018-09-25 21:08:41 +03:00
|
|
|
/* forward declare support classes from mercurial */
|
|
|
|
class ConstantStringRef;
|
|
|
|
class DatapackStore;
|
|
|
|
class UnionDatapackStore;
|
|
|
|
|
2016-06-14 01:15:32 +03:00
|
|
|
namespace facebook {
|
|
|
|
namespace eden {
|
|
|
|
|
2019-09-20 06:21:32 +03:00
|
|
|
class HgImporter;
|
2019-05-10 19:21:53 +03:00
|
|
|
struct ImporterOptions;
|
2016-06-14 01:15:32 +03:00
|
|
|
class LocalStore;
|
2019-04-08 23:45:35 +03:00
|
|
|
class MononokeHttpBackingStore;
|
|
|
|
class MononokeThriftBackingStore;
|
|
|
|
class MononokeCurlBackingStore;
|
2018-08-03 23:08:37 +03:00
|
|
|
class UnboundedQueueExecutor;
|
2018-10-31 21:48:12 +03:00
|
|
|
class ReloadableConfig;
|
2019-04-12 21:17:47 +03:00
|
|
|
class ServiceAddress;
|
2016-06-14 01:15:32 +03:00
|
|
|
|
|
|
|
/**
|
|
|
|
* A BackingStore implementation that loads data out of a mercurial repository.
|
|
|
|
*/
|
|
|
|
class HgBackingStore : public BackingStore {
|
|
|
|
public:
|
|
|
|
/**
|
|
|
|
* Create a new HgBackingStore.
|
|
|
|
*
|
|
|
|
* The LocalStore object is owned by the EdenServer (which also owns this
|
|
|
|
* HgBackingStore object). It is guaranteed to be valid for the lifetime of
|
|
|
|
* the HgBackingStore object.
|
|
|
|
*/
|
2017-11-15 20:46:11 +03:00
|
|
|
HgBackingStore(
|
|
|
|
AbsolutePathPiece repository,
|
|
|
|
LocalStore* localStore,
|
2018-08-20 21:07:34 +03:00
|
|
|
UnboundedQueueExecutor* serverThreadPool,
|
2019-04-11 05:55:31 +03:00
|
|
|
std::shared_ptr<ReloadableConfig> config,
|
2019-04-15 06:42:26 +03:00
|
|
|
std::shared_ptr<EdenStats>);
|
2018-06-07 00:32:41 +03:00
|
|
|
|
|
|
|
/**
|
|
|
|
* Create an HgBackingStore suitable for use in unit tests. It uses an inline
|
|
|
|
* executor to process loaded objects rather than the thread pools used in
|
|
|
|
* production Eden.
|
|
|
|
*/
|
2019-05-28 08:30:08 +03:00
|
|
|
HgBackingStore(
|
2019-09-20 06:21:32 +03:00
|
|
|
AbsolutePathPiece repository,
|
|
|
|
HgImporter* importer,
|
2019-05-28 08:30:08 +03:00
|
|
|
LocalStore* localStore,
|
|
|
|
std::shared_ptr<EdenStats>);
|
2018-06-07 00:32:41 +03:00
|
|
|
|
2017-05-24 02:08:03 +03:00
|
|
|
~HgBackingStore() override;
|
2016-06-14 01:15:32 +03:00
|
|
|
|
2019-12-19 03:14:07 +03:00
|
|
|
folly::SemiFuture<std::unique_ptr<Tree>> getTree(const Hash& id) override;
|
2019-12-19 03:14:07 +03:00
|
|
|
folly::SemiFuture<std::unique_ptr<Blob>> getBlob(const Hash& id) override;
|
2019-12-19 03:14:07 +03:00
|
|
|
folly::SemiFuture<std::unique_ptr<Tree>> getTreeForCommit(
|
2016-12-14 05:11:05 +03:00
|
|
|
const Hash& commitID) override;
|
2019-12-19 03:14:07 +03:00
|
|
|
folly::SemiFuture<std::unique_ptr<Tree>> getTreeForManifest(
|
2019-12-10 03:23:34 +03:00
|
|
|
const Hash& commitID,
|
|
|
|
const Hash& manifestID) override;
|
2018-06-15 07:32:45 +03:00
|
|
|
FOLLY_NODISCARD folly::Future<folly::Unit> prefetchBlobs(
|
2018-05-25 23:47:49 +03:00
|
|
|
const std::vector<Hash>& ids) const override;
|
2018-08-24 00:12:10 +03:00
|
|
|
|
2020-01-18 01:57:31 +03:00
|
|
|
void periodicManagementTask() override;
|
|
|
|
|
2018-09-25 21:08:41 +03:00
|
|
|
/**
|
|
|
|
* Import the manifest for the specified revision using mercurial
|
|
|
|
* treemanifest data.
|
|
|
|
*/
|
2019-12-20 00:35:55 +03:00
|
|
|
folly::Future<std::unique_ptr<Tree>> importTreeManifest(const Hash& commitId);
|
2018-09-25 21:08:41 +03:00
|
|
|
|
2016-06-14 01:15:32 +03:00
|
|
|
private:
|
|
|
|
// Forbidden copy constructor and assignment operator
|
|
|
|
HgBackingStore(HgBackingStore const&) = delete;
|
|
|
|
HgBackingStore& operator=(HgBackingStore const&) = delete;
|
|
|
|
|
2018-09-25 21:08:41 +03:00
|
|
|
/**
|
|
|
|
* Initialize the unionStore_ needed for treemanifest import support.
|
|
|
|
*/
|
|
|
|
void initializeTreeManifestImport(
|
|
|
|
const ImporterOptions& options,
|
|
|
|
AbsolutePathPiece repoPath);
|
|
|
|
|
|
|
|
/**
|
2019-04-08 23:45:35 +03:00
|
|
|
* Create a Mononoke backing store based on config_.
|
2018-09-25 21:08:41 +03:00
|
|
|
*
|
2019-04-08 23:45:35 +03:00
|
|
|
* Return nullptr if something is wrong (e.g. missing configs).
|
2018-09-25 21:08:41 +03:00
|
|
|
*/
|
2019-04-08 23:45:35 +03:00
|
|
|
std::unique_ptr<BackingStore> initializeMononoke();
|
|
|
|
|
|
|
|
/**
|
|
|
|
* Get an instace of Mononoke backing store as specified in config_. This will
|
|
|
|
* call `initializeMononoke` if no active Mononoke instance is stored.
|
|
|
|
*
|
|
|
|
* Return nullptr if Mononoke is disabled.
|
|
|
|
*/
|
|
|
|
std::shared_ptr<BackingStore> getMononoke();
|
2018-09-25 21:08:41 +03:00
|
|
|
|
2019-04-12 21:17:47 +03:00
|
|
|
/**
|
|
|
|
* Get an instance of `ServiceAddress` that points to Mononoke API Server
|
|
|
|
* based on user's configuration. It could be a pair of host and port or a smc
|
|
|
|
* tier name.
|
|
|
|
*/
|
|
|
|
std::unique_ptr<ServiceAddress> getMononokeServiceAddress();
|
|
|
|
|
2019-08-28 03:02:49 +03:00
|
|
|
#if EDEN_HAVE_MONONOKE
|
2019-03-12 00:20:44 +03:00
|
|
|
/**
|
2019-04-08 23:45:35 +03:00
|
|
|
* Create an instance of MononokeHttpBackingStore with values from config_
|
|
|
|
* (Proxygen based Mononoke client)
|
2019-03-12 00:20:44 +03:00
|
|
|
*
|
2019-04-08 23:45:35 +03:00
|
|
|
* Return null if SSLContext cannot be constructed.
|
2019-03-12 00:20:44 +03:00
|
|
|
*/
|
2019-04-08 23:45:35 +03:00
|
|
|
std::unique_ptr<MononokeHttpBackingStore>
|
|
|
|
initializeHttpMononokeBackingStore();
|
2019-03-26 23:44:08 +03:00
|
|
|
|
|
|
|
/**
|
2019-04-08 23:45:35 +03:00
|
|
|
* Create an instance of MononokeThriftBackingStore with values from config_
|
|
|
|
* (Thrift based Mononoke client)
|
|
|
|
*
|
|
|
|
* Return nullptr if required config is missing.
|
2019-03-26 23:44:08 +03:00
|
|
|
*/
|
2019-04-08 23:45:35 +03:00
|
|
|
std::unique_ptr<MononokeThriftBackingStore>
|
|
|
|
initializeThriftMononokeBackingStore();
|
2019-03-12 00:20:44 +03:00
|
|
|
#endif
|
2019-03-12 00:20:44 +03:00
|
|
|
|
2019-05-02 22:27:27 +03:00
|
|
|
#if defined(EDEN_HAVE_CURL)
|
2019-03-12 00:20:44 +03:00
|
|
|
/**
|
2019-04-08 23:45:35 +03:00
|
|
|
* Create an instance of MononokeCurlBackingStore with values from config_
|
|
|
|
* (Curl based Mononoke client)
|
|
|
|
*
|
|
|
|
* Return nullptr if required config is missing.
|
2019-03-12 00:20:44 +03:00
|
|
|
*/
|
2019-04-08 23:45:35 +03:00
|
|
|
std::unique_ptr<MononokeCurlBackingStore>
|
|
|
|
initializeCurlMononokeBackingStore();
|
2018-10-31 21:48:13 +03:00
|
|
|
#endif
|
|
|
|
|
2019-12-19 03:14:07 +03:00
|
|
|
folly::SemiFuture<std::unique_ptr<Blob>> getBlobFromHgImporter(
|
2020-01-16 23:43:54 +03:00
|
|
|
const RelativePathPiece& path,
|
2019-12-19 03:14:07 +03:00
|
|
|
const Hash& id);
|
2019-05-28 07:51:16 +03:00
|
|
|
|
2018-09-25 21:08:41 +03:00
|
|
|
folly::Future<std::unique_ptr<Tree>> getTreeForCommitImpl(Hash commitID);
|
2016-12-14 05:11:05 +03:00
|
|
|
|
2019-12-10 03:23:34 +03:00
|
|
|
folly::Future<std::unique_ptr<Tree>> getTreeForRootTreeImpl(
|
|
|
|
const Hash& commitID,
|
|
|
|
const Hash& rootTreeHash);
|
|
|
|
|
2018-07-09 21:57:54 +03:00
|
|
|
// Import the Tree from Hg and cache it in the LocalStore before returning it.
|
2019-12-19 03:14:07 +03:00
|
|
|
folly::SemiFuture<std::unique_ptr<Tree>> importTreeForCommit(Hash commitID);
|
2018-09-25 21:08:41 +03:00
|
|
|
|
2018-11-01 03:56:05 +03:00
|
|
|
void initializeDatapackImport(AbsolutePathPiece repository);
|
2018-09-25 21:08:41 +03:00
|
|
|
folly::Future<std::unique_ptr<Tree>> importTreeImpl(
|
|
|
|
const Hash& manifestNode,
|
|
|
|
const Hash& edenTreeID,
|
2019-06-20 03:41:24 +03:00
|
|
|
RelativePathPiece path);
|
2018-10-31 21:48:14 +03:00
|
|
|
folly::Future<std::unique_ptr<Tree>> fetchTreeFromHgCacheOrImporter(
|
|
|
|
Hash manifestNode,
|
|
|
|
Hash edenTreeID,
|
2019-06-20 03:41:24 +03:00
|
|
|
RelativePath path);
|
2018-09-25 21:08:41 +03:00
|
|
|
folly::Future<std::unique_ptr<Tree>> fetchTreeFromImporter(
|
|
|
|
Hash manifestNode,
|
|
|
|
Hash edenTreeID,
|
|
|
|
RelativePath path,
|
|
|
|
std::shared_ptr<LocalStore::WriteBatch> writeBatch);
|
|
|
|
std::unique_ptr<Tree> processTree(
|
|
|
|
ConstantStringRef& content,
|
|
|
|
const Hash& manifestNode,
|
|
|
|
const Hash& edenTreeID,
|
|
|
|
RelativePathPiece path,
|
|
|
|
LocalStore::WriteBatch* writeBatch);
|
|
|
|
|
2016-06-14 01:15:32 +03:00
|
|
|
LocalStore* localStore_{nullptr};
|
2019-04-17 23:07:08 +03:00
|
|
|
std::shared_ptr<EdenStats> stats_;
|
2017-11-15 20:46:11 +03:00
|
|
|
// A set of threads owning HgImporter instances
|
|
|
|
std::unique_ptr<folly::Executor> importThreadPool_;
|
2018-10-31 21:48:12 +03:00
|
|
|
std::shared_ptr<ReloadableConfig> config_;
|
2017-11-15 20:46:11 +03:00
|
|
|
// The main server thread pool; we push the Futures back into
|
|
|
|
// this pool to run their completion code to avoid clogging
|
2017-12-12 23:23:57 +03:00
|
|
|
// the importer pool. Queuing in this pool can never block (which would risk
|
|
|
|
// deadlock) or throw an exception when full (which would incorrectly fail the
|
|
|
|
// load).
|
2018-06-07 00:32:41 +03:00
|
|
|
folly::Executor* serverThreadPool_;
|
2019-05-02 22:27:27 +03:00
|
|
|
|
2018-09-25 21:08:41 +03:00
|
|
|
// These DatapackStore objects are never referenced once UnionDatapackStore
|
|
|
|
// is allocated. They are here solely so their lifetime persists while the
|
|
|
|
// UnionDatapackStore is alive.
|
|
|
|
std::vector<std::unique_ptr<DatapackStore>> dataPackStores_;
|
|
|
|
std::unique_ptr<folly::Synchronized<UnionDatapackStore>> unionStore_;
|
|
|
|
|
2019-04-08 23:45:35 +03:00
|
|
|
std::string repoName_;
|
|
|
|
folly::Synchronized<std::shared_ptr<BackingStore>> mononoke_;
|
2019-10-09 23:00:50 +03:00
|
|
|
#ifdef EDEN_HAVE_RUST_DATAPACK
|
2019-10-01 23:11:16 +03:00
|
|
|
std::optional<HgDatapackStore> datapackStore_;
|
2018-11-01 03:56:05 +03:00
|
|
|
#endif
|
2020-03-05 19:59:28 +03:00
|
|
|
|
|
|
|
mutable std::atomic<size_t> pendingImportBlobCount_{0};
|
|
|
|
mutable std::atomic<size_t> pendingImportTreeCount_{0};
|
|
|
|
mutable std::atomic<size_t> pendingImportPrefetchCount_{0};
|
2016-06-14 01:15:32 +03:00
|
|
|
};
|
2017-11-04 01:58:04 +03:00
|
|
|
} // namespace eden
|
|
|
|
} // namespace facebook
|