2016-05-12 23:43:17 +03:00
|
|
|
/*
|
2017-01-21 09:02:33 +03:00
|
|
|
* Copyright (c) 2016-present, Facebook, Inc.
|
2016-05-12 23:43:17 +03:00
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* This source code is licensed under the BSD-style license found in the
|
|
|
|
* LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
* of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
*
|
|
|
|
*/
|
|
|
|
#include "LocalStore.h"
|
2016-06-09 00:52:51 +03:00
|
|
|
|
2016-05-12 23:43:17 +03:00
|
|
|
#include <folly/Format.h>
|
|
|
|
#include <folly/String.h>
|
2018-05-10 04:41:21 +03:00
|
|
|
#include <folly/futures/Future.h>
|
2016-06-16 00:23:24 +03:00
|
|
|
#include <folly/io/Cursor.h>
|
|
|
|
#include <folly/io/IOBuf.h>
|
2017-12-08 11:24:44 +03:00
|
|
|
#include <folly/lang/Bits.h>
|
2018-05-01 07:20:51 +03:00
|
|
|
#include <folly/logging/xlog.h>
|
2016-05-12 23:43:17 +03:00
|
|
|
#include <array>
|
2017-06-22 23:39:57 +03:00
|
|
|
|
2016-06-09 04:59:56 +03:00
|
|
|
#include "eden/fs/model/Blob.h"
|
|
|
|
#include "eden/fs/model/Tree.h"
|
2016-05-12 23:43:17 +03:00
|
|
|
#include "eden/fs/model/git/GitBlob.h"
|
|
|
|
#include "eden/fs/model/git/GitTree.h"
|
2018-10-31 21:48:14 +03:00
|
|
|
#include "eden/fs/store/SerializedBlobMetadata.h"
|
2016-06-09 04:59:56 +03:00
|
|
|
#include "eden/fs/store/StoreResult.h"
|
2016-05-12 23:43:17 +03:00
|
|
|
|
|
|
|
using facebook::eden::Hash;
|
|
|
|
using folly::ByteRange;
|
2016-06-16 00:23:24 +03:00
|
|
|
using folly::IOBuf;
|
2016-05-12 23:43:17 +03:00
|
|
|
using folly::StringPiece;
|
2017-11-04 01:58:04 +03:00
|
|
|
using folly::io::Cursor;
|
2018-10-24 03:03:12 +03:00
|
|
|
using std::optional;
|
2016-05-12 23:43:17 +03:00
|
|
|
using std::string;
|
|
|
|
using std::unique_ptr;
|
|
|
|
|
2016-06-09 00:52:51 +03:00
|
|
|
namespace {
|
2016-12-13 04:48:46 +03:00
|
|
|
using namespace facebook::eden;
|
2018-08-10 21:09:48 +03:00
|
|
|
enum class Persistence : bool {
|
|
|
|
Ephemeral = false,
|
|
|
|
Persistent = true,
|
|
|
|
};
|
|
|
|
|
|
|
|
static constexpr struct KeySpaceRecord {
|
|
|
|
LocalStore::KeySpace keySpace;
|
|
|
|
Persistence persistence;
|
|
|
|
} kKeySpaceRecords[] = {
|
|
|
|
{LocalStore::BlobFamily, Persistence::Ephemeral},
|
|
|
|
{LocalStore::BlobMetaDataFamily, Persistence::Ephemeral},
|
|
|
|
|
|
|
|
// If the trees were imported from a flatmanifest, we cannot delete them.
|
|
|
|
// See test_contents_are_the_same_if_handle_is_held_open when running
|
|
|
|
// against a flatmanifest repository.
|
|
|
|
{LocalStore::TreeFamily, Persistence::Persistent},
|
|
|
|
|
|
|
|
// Proxy hashes are required to fetch objects from hg from a hash.
|
|
|
|
// Deleting them breaks re-importing after an inode is unloaded.
|
|
|
|
{LocalStore::HgProxyHashFamily, Persistence::Persistent},
|
|
|
|
|
|
|
|
{LocalStore::HgCommitToTreeFamily, Persistence::Ephemeral},
|
|
|
|
};
|
2017-11-04 01:58:04 +03:00
|
|
|
} // namespace
|
2016-05-12 23:43:17 +03:00
|
|
|
|
|
|
|
namespace facebook {
|
|
|
|
namespace eden {
|
|
|
|
|
2018-11-09 22:20:16 +03:00
|
|
|
LocalStore::LocalStore(std::shared_ptr<ReloadableConfig> config) noexcept
|
|
|
|
: config_(std::move(config)) {}
|
|
|
|
|
2018-08-10 21:09:48 +03:00
|
|
|
void LocalStore::clearCachesAndCompactAll() {
|
|
|
|
for (auto ks : kKeySpaceRecords) {
|
|
|
|
if (ks.persistence == Persistence::Ephemeral) {
|
|
|
|
clearKeySpace(ks.keySpace);
|
|
|
|
}
|
|
|
|
compactKeySpace(ks.keySpace);
|
|
|
|
}
|
|
|
|
}
|
2018-05-31 20:39:53 +03:00
|
|
|
|
2018-08-10 21:09:48 +03:00
|
|
|
void LocalStore::clearCaches() {
|
|
|
|
for (auto ks : kKeySpaceRecords) {
|
|
|
|
if (ks.persistence == Persistence::Ephemeral) {
|
|
|
|
clearKeySpace(ks.keySpace);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2018-05-31 20:39:53 +03:00
|
|
|
|
2018-08-10 21:09:48 +03:00
|
|
|
void LocalStore::compactStorage() {
|
|
|
|
for (auto ks : kKeySpaceRecords) {
|
|
|
|
compactKeySpace(ks.keySpace);
|
|
|
|
}
|
2018-05-31 20:39:49 +03:00
|
|
|
}
|
|
|
|
|
2017-09-08 00:39:47 +03:00
|
|
|
StoreResult LocalStore::get(KeySpace keySpace, const Hash& id) const {
|
|
|
|
return get(keySpace, id.getBytes());
|
2016-05-12 23:43:17 +03:00
|
|
|
}
|
|
|
|
|
2018-05-10 04:41:21 +03:00
|
|
|
// This is the fallback implementation for stores that don't have any
|
|
|
|
// internal support for asynchronous fetches. This just performs the
|
|
|
|
// fetch and wraps it in a future
|
|
|
|
folly::Future<StoreResult> LocalStore::getFuture(
|
|
|
|
KeySpace keySpace,
|
|
|
|
folly::ByteRange key) const {
|
|
|
|
return folly::makeFutureWith(
|
|
|
|
[keySpace, key, this] { return get(keySpace, key); });
|
|
|
|
}
|
|
|
|
|
2018-05-25 23:47:54 +03:00
|
|
|
folly::Future<std::vector<StoreResult>> LocalStore::getBatch(
|
|
|
|
KeySpace keySpace,
|
|
|
|
const std::vector<folly::ByteRange>& keys) const {
|
|
|
|
return folly::makeFutureWith([keySpace, keys, this] {
|
|
|
|
std::vector<StoreResult> results;
|
|
|
|
for (auto& key : keys) {
|
|
|
|
results.emplace_back(get(keySpace, key));
|
|
|
|
}
|
|
|
|
return results;
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2016-05-12 23:43:17 +03:00
|
|
|
// TODO(mbolin): Currently, all objects in our RocksDB are Git objects. We
|
|
|
|
// probably want to namespace these by column family going forward, at which
|
|
|
|
// point we might want to have a GitLocalStore that delegates to an
|
|
|
|
// LocalStore so a vanilla LocalStore has no knowledge of deserializeGitTree()
|
|
|
|
// or deserializeGitBlob().
|
|
|
|
|
2018-06-01 07:05:09 +03:00
|
|
|
folly::Future<std::unique_ptr<Tree>> LocalStore::getTree(const Hash& id) const {
|
2018-05-10 04:41:21 +03:00
|
|
|
return getFuture(KeySpace::TreeFamily, id.getBytes())
|
2018-09-07 21:02:45 +03:00
|
|
|
.thenValue([id](StoreResult&& data) {
|
2018-05-10 04:41:21 +03:00
|
|
|
if (!data.isValid()) {
|
|
|
|
return std::unique_ptr<Tree>(nullptr);
|
|
|
|
}
|
|
|
|
return deserializeGitTree(id, data.bytes());
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2018-06-01 06:40:09 +03:00
|
|
|
folly::Future<std::unique_ptr<Blob>> LocalStore::getBlob(const Hash& id) const {
|
2018-05-10 04:41:21 +03:00
|
|
|
return getFuture(KeySpace::BlobFamily, id.getBytes())
|
2018-09-07 21:02:45 +03:00
|
|
|
.thenValue([id](StoreResult&& data) {
|
2018-05-10 04:41:21 +03:00
|
|
|
if (!data.isValid()) {
|
|
|
|
return std::unique_ptr<Blob>(nullptr);
|
|
|
|
}
|
|
|
|
auto buf = data.extractIOBuf();
|
|
|
|
return deserializeGitBlob(id, &buf);
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2018-10-24 03:03:12 +03:00
|
|
|
folly::Future<optional<BlobMetadata>> LocalStore::getBlobMetadata(
|
2018-06-01 19:23:33 +03:00
|
|
|
const Hash& id) const {
|
|
|
|
return getFuture(KeySpace::BlobMetaDataFamily, id.getBytes())
|
2018-10-24 03:03:12 +03:00
|
|
|
.thenValue([id](StoreResult&& data) -> optional<BlobMetadata> {
|
2018-06-01 19:23:33 +03:00
|
|
|
if (!data.isValid()) {
|
2018-10-24 03:03:12 +03:00
|
|
|
return std::nullopt;
|
2018-06-01 19:23:33 +03:00
|
|
|
} else {
|
|
|
|
return SerializedBlobMetadata::parse(id, data);
|
|
|
|
}
|
|
|
|
});
|
2016-12-13 04:48:46 +03:00
|
|
|
}
|
2016-05-12 23:43:17 +03:00
|
|
|
|
2017-09-29 06:45:57 +03:00
|
|
|
std::pair<Hash, folly::IOBuf> LocalStore::serializeTree(const Tree* tree) {
|
|
|
|
GitTreeSerializer serializer;
|
|
|
|
for (auto& entry : tree->getTreeEntries()) {
|
|
|
|
serializer.addEntry(std::move(entry));
|
|
|
|
}
|
|
|
|
IOBuf treeBuf = serializer.finalize();
|
|
|
|
|
|
|
|
auto id = tree->getHash();
|
|
|
|
if (id == Hash()) {
|
2018-11-29 03:07:04 +03:00
|
|
|
id = Hash::sha1(treeBuf);
|
2017-09-29 06:45:57 +03:00
|
|
|
}
|
|
|
|
return std::make_pair(id, treeBuf);
|
|
|
|
}
|
|
|
|
|
|
|
|
bool LocalStore::hasKey(KeySpace keySpace, const Hash& id) const {
|
|
|
|
return hasKey(keySpace, id.getBytes());
|
|
|
|
}
|
|
|
|
|
|
|
|
Hash LocalStore::putTree(const Tree* tree) {
|
|
|
|
auto serialized = LocalStore::serializeTree(tree);
|
|
|
|
ByteRange treeData = serialized.second.coalesce();
|
|
|
|
|
|
|
|
auto& id = serialized.first;
|
|
|
|
put(KeySpace::TreeFamily, id, treeData);
|
|
|
|
return id;
|
|
|
|
}
|
|
|
|
|
|
|
|
Hash LocalStore::WriteBatch::putTree(const Tree* tree) {
|
|
|
|
auto serialized = LocalStore::serializeTree(tree);
|
|
|
|
ByteRange treeData = serialized.second.coalesce();
|
|
|
|
|
|
|
|
auto& id = serialized.first;
|
|
|
|
put(KeySpace::TreeFamily, id.getBytes(), treeData);
|
|
|
|
return id;
|
|
|
|
}
|
|
|
|
|
2016-12-13 04:48:46 +03:00
|
|
|
BlobMetadata LocalStore::putBlob(const Hash& id, const Blob* blob) {
|
2017-09-29 06:45:57 +03:00
|
|
|
// Since blob serialization is moderately complex, just delegate
|
|
|
|
// the immediate putBlob to the method on the WriteBatch.
|
|
|
|
// Pre-allocate a buffer of approximately the right size; it
|
|
|
|
// needs to hold the blob content plus have room for a couple of
|
|
|
|
// hashes for the keys, plus some padding.
|
|
|
|
auto batch = beginWrite(blob->getContents().computeChainDataLength() + 64);
|
2018-02-07 22:45:41 +03:00
|
|
|
auto result = batch->putBlob(id, blob);
|
|
|
|
batch->flush();
|
2017-09-29 06:45:57 +03:00
|
|
|
return result;
|
|
|
|
}
|
|
|
|
|
2018-02-07 22:45:41 +03:00
|
|
|
void LocalStore::put(
|
|
|
|
LocalStore::KeySpace keySpace,
|
|
|
|
const Hash& id,
|
|
|
|
folly::ByteRange value) {
|
|
|
|
put(keySpace, id.getBytes(), value);
|
|
|
|
}
|
|
|
|
|
|
|
|
void LocalStore::WriteBatch::put(
|
|
|
|
LocalStore::KeySpace keySpace,
|
|
|
|
const Hash& id,
|
|
|
|
folly::ByteRange value) {
|
|
|
|
put(keySpace, id.getBytes(), value);
|
|
|
|
}
|
|
|
|
|
2017-09-29 06:45:57 +03:00
|
|
|
BlobMetadata LocalStore::WriteBatch::putBlob(const Hash& id, const Blob* blob) {
|
2016-06-16 00:23:24 +03:00
|
|
|
const IOBuf& contents = blob->getContents();
|
|
|
|
|
2018-11-29 03:07:04 +03:00
|
|
|
BlobMetadata metadata{Hash::sha1(contents),
|
2016-12-13 04:48:46 +03:00
|
|
|
contents.computeChainDataLength()};
|
2017-03-31 21:16:47 +03:00
|
|
|
|
2016-12-13 04:48:46 +03:00
|
|
|
SerializedBlobMetadata metadataBytes(metadata);
|
2016-06-16 00:23:24 +03:00
|
|
|
|
2018-02-07 22:45:41 +03:00
|
|
|
auto hashSlice = id.getBytes();
|
2016-06-16 00:23:24 +03:00
|
|
|
|
|
|
|
// Add a git-style blob prefix
|
|
|
|
auto prefix = folly::to<string>("blob ", contents.computeChainDataLength());
|
|
|
|
prefix.push_back('\0');
|
2018-02-07 22:45:41 +03:00
|
|
|
std::vector<ByteRange> bodySlices;
|
|
|
|
bodySlices.emplace_back(StringPiece(prefix));
|
2016-06-16 00:23:24 +03:00
|
|
|
|
|
|
|
// Add all of the IOBuf chunks
|
|
|
|
Cursor cursor(&contents);
|
|
|
|
while (true) {
|
|
|
|
auto bytes = cursor.peekBytes();
|
|
|
|
if (bytes.empty()) {
|
|
|
|
break;
|
|
|
|
}
|
2018-02-07 22:45:41 +03:00
|
|
|
bodySlices.push_back(bytes);
|
2016-06-16 00:23:24 +03:00
|
|
|
cursor.skip(bytes.size());
|
|
|
|
}
|
|
|
|
|
2018-02-07 22:45:41 +03:00
|
|
|
put(LocalStore::KeySpace::BlobFamily, hashSlice, bodySlices);
|
|
|
|
put(LocalStore::KeySpace::BlobMetaDataFamily,
|
2017-09-29 06:45:57 +03:00
|
|
|
hashSlice,
|
|
|
|
metadataBytes.slice());
|
2016-12-13 04:48:46 +03:00
|
|
|
return metadata;
|
2016-05-12 23:43:17 +03:00
|
|
|
}
|
|
|
|
|
2018-02-07 22:45:41 +03:00
|
|
|
LocalStore::WriteBatch::~WriteBatch() {}
|
|
|
|
LocalStore::~LocalStore() {}
|
2016-05-12 23:43:17 +03:00
|
|
|
|
2017-09-29 06:45:57 +03:00
|
|
|
} // namespace eden
|
|
|
|
} // namespace facebook
|