facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-07 07:17:55 +03:00
Code Issues Projects Releases Wiki Activity

split RocksDbLocalStore out from LocalStore

Browse Source 
Summary:
This enables dropping in alternative implementations
of LocalStore and adds a MemoryLocalStore implementation for
use in our tests.

This diff doesn't change the default storage option for the
eden server.  I'll look at adding such an option in a follow up diff.

Reviewed By: chadaustin

Differential Revision: D6910413

fbshipit-source-id: 018bf04e0bff101e1f0ab35e8580ca2a2622e5ef
This commit is contained in:
Wez Furlong 2018-02-07 11:45:41 -08:00 committed by Facebook Github Bot
parent 4fb0ac3809
commit a0fb6d9d05
16 changed files with 569 additions and 296 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
eden/fs/service/EdenServer.cpp
View File

@ -35,6 +35,7 @@
#include "eden/fs/store/EmptyBackingStore.h"
#include "eden/fs/store/LocalStore.h"
#include "eden/fs/store/ObjectStore.h"
#include "eden/fs/store/RocksDbLocalStore.h"
#include "eden/fs/store/git/GitBackingStore.h"
#include "eden/fs/store/hg/HgBackingStore.h"
#include "eden/fs/takeover/TakeoverClient.h"
@ -302,7 +303,7 @@ void EdenServer::prepare() {
XLOG(DBG2) << "opening local RocksDB store";
const auto rocksPath = edenDir_ + RelativePathPiece{kRocksDBPath};
localStore_ = make_shared<LocalStore>(rocksPath);
localStore_ = make_shared<RocksDbLocalStore>(rocksPath);
XLOG(DBG2) << "done opening local RocksDB store";
// Start listening for graceful takeover requests

244
eden/fs/store/LocalStore.cpp
View File

@ -16,17 +16,12 @@
#include <folly/io/Cursor.h>
#include <folly/io/IOBuf.h>
#include <folly/lang/Bits.h>
#include <rocksdb/db.h>
#include <rocksdb/filter_policy.h>
#include <rocksdb/table.h>
#include <array>
#include "eden/fs/model/Blob.h"
#include "eden/fs/model/Tree.h"
#include "eden/fs/model/git/GitBlob.h"
#include "eden/fs/model/git/GitTree.h"
#include "eden/fs/rocksdb/RocksException.h"
#include "eden/fs/rocksdb/RocksHandles.h"
#include "eden/fs/store/StoreResult.h"
using facebook::eden::Hash;
@ -35,54 +30,11 @@ using folly::IOBuf;
using folly::Optional;
using folly::StringPiece;
using folly::io::Cursor;
using rocksdb::ReadOptions;
using rocksdb::Slice;
using rocksdb::SliceParts;
using rocksdb::WriteBatch;
using rocksdb::WriteOptions;
using std::string;
using std::unique_ptr;
namespace {
using namespace facebook::eden;
rocksdb::ColumnFamilyOptions makeColumnOptions(uint64_t LRUblockCacheSizeMB) {
rocksdb::ColumnFamilyOptions options;
// We'll never perform range scans on any of the keys that we store.
// This enables bloom filters and a hash policy that improves our
// get/put performance.
options.OptimizeForPointLookup(LRUblockCacheSizeMB);
options.OptimizeLevelStyleCompaction();
return options;
}
/**
* The different key spaces that we desire.
* The ordering is coupled with the values of the LocalStore::KeySpace enum.
*/
const std::vector<rocksdb::ColumnFamilyDescriptor>& columnFamilies() {
// Most of the column families will share the same cache. We
// want the blob data to live in its own smaller cache; the assumption
// is that the vfs cache will compensate for that, together with the
// idea that we shouldn't need to materialize a great many files.
auto options = makeColumnOptions(64);
auto blobOptions = makeColumnOptions(8);
// Meyers singleton to avoid SIOF issues
static const std::vector<rocksdb::ColumnFamilyDescriptor> families{
rocksdb::ColumnFamilyDescriptor{rocksdb::kDefaultColumnFamilyName,
options},
rocksdb::ColumnFamilyDescriptor{"blob", blobOptions},
rocksdb::ColumnFamilyDescriptor{"blobmeta", options},
rocksdb::ColumnFamilyDescriptor{"tree", options},
rocksdb::ColumnFamilyDescriptor{"hgproxyhash", options},
rocksdb::ColumnFamilyDescriptor{"hgcommit2tree", options},
};
return families;
}
class SerializedBlobMetadata {
public:
explicit SerializedBlobMetadata(const BlobMetadata& metadata) {
@ -92,8 +44,8 @@ class SerializedBlobMetadata {
serialize(contentsHash, blobSize);
}
Slice slice() const {
return Slice{reinterpret_cast<const char*>(data_.data()), data_.size()};
ByteRange slice() const {
return ByteRange{data_};
}
static BlobMetadata parse(Hash blobID, const StoreResult& result) {
@ -131,62 +83,11 @@ class SerializedBlobMetadata {
*/
std::array<uint8_t, SIZE> data_;
};
rocksdb::Slice _createSlice(folly::ByteRange bytes) {
return Slice(reinterpret_cast<const char*>(bytes.data()), bytes.size());
}
} // namespace
namespace facebook {
namespace eden {
LocalStore::LocalStore(AbsolutePathPiece pathToRocksDb)
: dbHandles_(pathToRocksDb.stringPiece(), columnFamilies()) {}
LocalStore::~LocalStore() {
#ifdef FOLLY_SANITIZE_ADDRESS
// RocksDB has some race conditions around setting up and tearing down
// the threads that it uses to maintain the database. This manifests
// in our test harness, particularly in a test where we quickly mount
// and then unmount. We see this as an abort with the message:
// "pthread lock: Invalid Argument".
// My assumption is that we're shutting things down before rocks has
// completed initializing. This sleep call is present in the destructor
// to make it more likely that rocks is past that critical point and
// so that we can shutdown successfully.
/* sleep override */ sleep(1);
#endif
}
void LocalStore::close() {
dbHandles_.columns.clear();
dbHandles_.db.reset();
}
StoreResult LocalStore::get(KeySpace keySpace, ByteRange key) const {
string value;
auto status = dbHandles_.db.get()->Get(
ReadOptions(),
dbHandles_.columns[keySpace].get(),
_createSlice(key),
&value);
if (!status.ok()) {
if (status.IsNotFound()) {
// Return an empty StoreResult
return StoreResult();
}
// TODO: RocksDB can return a "TryAgain" error.
// Should we try again for the user, rather than re-throwing the error?
// We don't use RocksException::check(), since we don't want to waste our
// time computing the hex string of the key if we succeeded.
throw RocksException::build(
status, "failed to get ", folly::hexlify(key), " from local store");
}
return StoreResult(std::move(value));
}
StoreResult LocalStore::get(KeySpace keySpace, const Hash& id) const {
return get(keySpace, id.getBytes());
}
@ -238,47 +139,10 @@ std::pair<Hash, folly::IOBuf> LocalStore::serializeTree(const Tree* tree) {
return std::make_pair(id, treeBuf);
}
bool LocalStore::hasKey(KeySpace keySpace, folly::ByteRange key) const {
string value;
auto status = dbHandles_.db->Get(
ReadOptions(),
dbHandles_.columns[keySpace].get(),
_createSlice(key),
&value);
if (!status.ok()) {
if (status.IsNotFound()) {
return false;
}
// TODO: RocksDB can return a "TryAgain" error.
// Should we try again for the user, rather than re-throwing the error?
// We don't use RocksException::check(), since we don't want to waste our
// time computing the hex string of the key if we succeeded.
throw RocksException::build(
status, "failed to get ", folly::hexlify(key), " from local store");
}
return true;
}
bool LocalStore::hasKey(KeySpace keySpace, const Hash& id) const {
return hasKey(keySpace, id.getBytes());
}
LocalStore::WriteBatch LocalStore::beginWrite(size_t bufSize) {
return LocalStore::WriteBatch(dbHandles_, bufSize);
}
LocalStore::WriteBatch::WriteBatch(RocksHandles& dbHandles, size_t bufSize)
: dbHandles_(dbHandles), writeBatch_(bufSize), bufSize_(bufSize) {}
LocalStore::WriteBatch::~WriteBatch() {
if (writeBatch_.Count() > 0) {
XLOG(ERR) << "WriteBatch being destroyed with " << writeBatch_.Count()
<< " items pending flush";
}
}
Hash LocalStore::putTree(const Tree* tree) {
auto serialized = LocalStore::serializeTree(tree);
ByteRange treeData = serialized.second.coalesce();
@ -304,11 +168,25 @@ BlobMetadata LocalStore::putBlob(const Hash& id, const Blob* blob) {
// needs to hold the blob content plus have room for a couple of
// hashes for the keys, plus some padding.
auto batch = beginWrite(blob->getContents().computeChainDataLength() + 64);
auto result = batch.putBlob(id, blob);
batch.flush();
auto result = batch->putBlob(id, blob);
batch->flush();
return result;
}
void LocalStore::put(
LocalStore::KeySpace keySpace,
const Hash& id,
folly::ByteRange value) {
put(keySpace, id.getBytes(), value);
}
void LocalStore::WriteBatch::put(
LocalStore::KeySpace keySpace,
const Hash& id,
folly::ByteRange value) {
put(keySpace, id.getBytes(), value);
}
BlobMetadata LocalStore::WriteBatch::putBlob(const Hash& id, const Blob* blob) {
const IOBuf& contents = blob->getContents();
@ -317,16 +195,14 @@ BlobMetadata LocalStore::WriteBatch::putBlob(const Hash& id, const Blob* blob) {
SerializedBlobMetadata metadataBytes(metadata);
auto hashSlice = _createSlice(id.getBytes());
SliceParts keyParts(&hashSlice, 1);
auto hashSlice = id.getBytes();
ByteRange bodyBytes;
// Add a git-style blob prefix
auto prefix = folly::to<string>("blob ", contents.computeChainDataLength());
prefix.push_back('\0');
std::vector<Slice> bodySlices;
bodySlices.emplace_back(prefix);
std::vector<ByteRange> bodySlices;
bodySlices.emplace_back(StringPiece(prefix));
// Add all of the IOBuf chunks
Cursor cursor(&contents);
@ -335,87 +211,19 @@ BlobMetadata LocalStore::WriteBatch::putBlob(const Hash& id, const Blob* blob) {
if (bytes.empty()) {
break;
}
bodySlices.push_back(_createSlice(bytes));
bodySlices.push_back(bytes);
cursor.skip(bytes.size());
}
SliceParts bodyParts(bodySlices.data(), bodySlices.size());
writeBatch_.Put(
dbHandles_.columns[KeySpace::BlobFamily].get(), keyParts, bodyParts);
writeBatch_.Put(
dbHandles_.columns[KeySpace::BlobMetaDataFamily].get(),
put(LocalStore::KeySpace::BlobFamily, hashSlice, bodySlices);
put(LocalStore::KeySpace::BlobMetaDataFamily,
hashSlice,
metadataBytes.slice());
flushIfNeeded();
return metadata;
}
void LocalStore::WriteBatch::flush() {
auto pending = writeBatch_.Count();
if (pending == 0) {
return;
}
XLOG(DBG5) << "Flushing " << pending << " entries with data size of "
<< writeBatch_.GetDataSize();
auto status = dbHandles_.db->Write(WriteOptions(), &writeBatch_);
XLOG(DBG5) << "... Flushed";
if (!status.ok()) {
throw RocksException::build(
status, "error putting blob batch in local store");
}
writeBatch_.Clear();
}
void LocalStore::WriteBatch::flushIfNeeded() {
auto needFlush = bufSize_ > 0 && writeBatch_.GetDataSize() >= bufSize_;
if (needFlush) {
flush();
}
}
void LocalStore::put(
LocalStore::KeySpace keySpace,
const Hash& id,
folly::ByteRange value) {
put(keySpace, id.getBytes(), value);
}
void LocalStore::put(
LocalStore::KeySpace keySpace,
folly::ByteRange key,
folly::ByteRange value) {
dbHandles_.db->Put(
WriteOptions(),
dbHandles_.columns[keySpace].get(),
_createSlice(key),
_createSlice(value));
}
void LocalStore::WriteBatch::put(
LocalStore::KeySpace keySpace,
const Hash& id,
folly::ByteRange value) {
put(keySpace, id.getBytes(), value);
}
void LocalStore::WriteBatch::put(
LocalStore::KeySpace keySpace,
folly::ByteRange key,
folly::ByteRange value) {
writeBatch_.Put(
dbHandles_.columns[keySpace].get(),
_createSlice(key),
_createSlice(value));
flushIfNeeded();
}
LocalStore::WriteBatch::~WriteBatch() {}
LocalStore::~LocalStore() {}
} // namespace eden
} // namespace facebook

72
eden/fs/store/LocalStore.h
View File

@ -34,23 +34,26 @@ class Tree;
* This is a content-addressed store, so objects can be only retrieved using
* their hash.
*
* The LocalStore is only a cache. If an object is not found in the LocalStore
* then it will need to be retrieved from the BackingStore.
*
* LocalStore uses RocksDB for the underlying storage.
* The LocalStore was originally only a cache. The intent was that If an
* object is not found in the LocalStore then it will need to be retrieved
* from the BackingStore. The introduction of HgProxyHashFamily renders this
* comment a little inaccurate because we don't have a way to produce the
* required data if the proxy hash data has been removed. We expect things
* to revert back to a more pure cache as we evolve our interfaces with
* Mercurial and Mononoke.
*
* LocalStore is thread-safe, and can be used from multiple threads without
* requiring the caller to perform locking around accesses to the LocalStore.
*/
class LocalStore {
public:
explicit LocalStore(AbsolutePathPiece pathToRocksDb);
virtual ~LocalStore();
/**
* Which key space (and thus column family) should be used to store
* a specific key. The values of these are coupled to the ordering
* of the columnFamilies descriptor in LocalStore.cpp. */
* Which key space (and thus column family for the RocksDbLocalStore)
* should be used to store a specific key. The values of these are
* coupled to the ordering of the columnFamilies descriptor in
* RocksDbLocalStore.cpp. */
enum KeySpace {
/* 0 is the default column family, which we are not using */
BlobFamily = 1,
@ -58,12 +61,14 @@ class LocalStore {
TreeFamily = 3,
HgProxyHashFamily = 4,
HgCommitToTreeFamily = 5,
End, // must be last!
};
/**
* Close the underlying RocksDB.
* Close the underlying store.
*/
void close();
virtual void close() = 0;
/**
* Get arbitrary unserialized data from the store.
@ -73,7 +78,7 @@ class LocalStore {
*
* May throw exceptions on error.
*/
StoreResult get(KeySpace keySpace, folly::ByteRange key) const;
virtual StoreResult get(KeySpace keySpace, folly::ByteRange key) const = 0;
StoreResult get(KeySpace keySpace, const Hash& id) const;
/**
@ -116,7 +121,7 @@ class LocalStore {
/**
* Test whether the key is stored.
*/
bool hasKey(KeySpace keySpace, folly::ByteRange key) const;
virtual bool hasKey(KeySpace keySpace, folly::ByteRange key) const = 0;
bool hasKey(KeySpace keySpace, const Hash& id) const;
/**
@ -137,11 +142,12 @@ class LocalStore {
/**
* Put arbitrary data in the store.
*/
void put(KeySpace keySpace, folly::ByteRange key, folly::ByteRange value);
virtual void
put(KeySpace keySpace, folly::ByteRange key, folly::ByteRange value) = 0;
void put(KeySpace keySpace, const Hash& id, folly::ByteRange value);
/*
* WriteBatch is a helper class that wraps RocksDB WriteBatch.
* WriteBatch is a helper class for facilitating a bulk store operation.
*
* The purpose of this class is to let multiple callers manage independent
* write batches and flush them to the backing storage when its deemed
@ -151,9 +157,9 @@ class LocalStore {
*
* Typical usage:
* auto writer = localStore->beginWrite();
* writer.put(KeySpace::Meta, Key, Value);
* writer.put(KeySpace::Blob, Key, BlobValue);
* writer.flush();
* writer->put(KeySpace::Meta, Key, Value);
* writer->put(KeySpace::Blob, Key, BlobValue);
* writer->flush();
*/
class WriteBatch {
public:
@ -175,31 +181,34 @@ class LocalStore {
/**
* Put arbitrary data in the store.
*/
void put(KeySpace keySpace, folly::ByteRange key, folly::ByteRange value);
virtual void
put(KeySpace keySpace, folly::ByteRange key, folly::ByteRange value) = 0;
void put(KeySpace keySpace, const Hash& id, folly::ByteRange value);
/**
* Put arbitrary data in the store where the value is split across
* a set of sliced data.
*/
virtual void put(
KeySpace keySpace,
folly::ByteRange key,
std::vector<folly::ByteRange> valueSlices) = 0;
/**
* Flush any pending data to the store.
*/
void flush();
virtual void flush() = 0;
// Forbidden copy construction/assignment; allow only moves
WriteBatch(const WriteBatch&) = delete;
WriteBatch(WriteBatch&&) = default;
WriteBatch& operator=(const WriteBatch&) = delete;
WriteBatch& operator=(WriteBatch&&) = default;
~WriteBatch();
virtual ~WriteBatch();
WriteBatch() = default;
private:
friend class LocalStore;
// Use LocalStore::beginWrite() to create a write batch
WriteBatch(RocksHandles& dbHandles, size_t bufferSize);
void flushIfNeeded();
RocksHandles& dbHandles_;
rocksdb::WriteBatch writeBatch_;
size_t bufSize_;
};
/**
@ -208,13 +217,10 @@ class LocalStore {
* the accumulated data exceeds bufSize. Otherwise no implifict flushing
* will occur.
* Either way, the caller will typically want to finish up by calling
* writeBatch.flush() to complete the batch as there is no implicit flush on
* writeBatch->flush() to complete the batch as there is no implicit flush on
* destruction either.
*/
WriteBatch beginWrite(size_t bufSize = 0);
private:
RocksHandles dbHandles_;
virtual std::unique_ptr<WriteBatch> beginWrite(size_t bufSize = 0) = 0;
};
} // namespace eden
} // namespace facebook

97
eden/fs/store/MemoryLocalStore.cpp Normal file
View File

@ -0,0 +1,97 @@
/*
* Copyright (c) 2018-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
#include "eden/fs/store/MemoryLocalStore.h"
#include <folly/String.h>
#include "eden/fs/store/StoreResult.h"
namespace facebook {
namespace eden {
using folly::StringPiece;
namespace {
class MemoryWriteBatch : public LocalStore::WriteBatch {
public:
explicit MemoryWriteBatch(MemoryLocalStore* store) : store_(store) {
storage_.resize(LocalStore::KeySpace::End);
}
void put(
LocalStore::KeySpace keySpace,
folly::ByteRange key,
folly::ByteRange value) override {
storage_[keySpace][StringPiece(key)] = StringPiece(value).str();
}
void put(
LocalStore::KeySpace keySpace,
folly::ByteRange key,
std::vector<folly::ByteRange> valueSlices) override {
std::string value;
for (const auto& slice : valueSlices) {
value.append(reinterpret_cast<const char*>(slice.data()), slice.size());
}
put(keySpace, key, StringPiece(value));
}
void flush() override {
for (size_t keySpace = 0; keySpace < storage_.size(); ++keySpace) {
for (const auto& it : storage_[keySpace]) {
store_->put(
static_cast<LocalStore::KeySpace>(keySpace),
it.first,
StringPiece(it.second));
}
storage_[keySpace].clear();
}
}
private:
MemoryLocalStore* store_;
std::vector<folly::StringKeyedUnorderedMap<std::string>> storage_;
};
} // namespace
MemoryLocalStore::MemoryLocalStore() {
storage_->resize(KeySpace::End);
}
void MemoryLocalStore::close() {}
StoreResult MemoryLocalStore::get(
LocalStore::KeySpace keySpace,
folly::ByteRange key) const {
auto store = storage_.rlock();
auto it = (*store)[keySpace].find(StringPiece(key));
if (it == (*store)[keySpace].end()) {
return StoreResult();
}
return StoreResult(std::string(it->second));
}
bool MemoryLocalStore::hasKey(
LocalStore::KeySpace keySpace,
folly::ByteRange key) const {
auto store = storage_.rlock();
auto it = (*store)[keySpace].find(StringPiece(key));
return it != (*store)[keySpace].end();
}
void MemoryLocalStore::put(
LocalStore::KeySpace keySpace,
folly::ByteRange key,
folly::ByteRange value) {
(*storage_.wlock())[keySpace][StringPiece(key)] = StringPiece(value).str();
}
std::unique_ptr<LocalStore::WriteBatch> MemoryLocalStore::beginWrite(size_t) {
return std::make_unique<MemoryWriteBatch>(this);
}
} // namespace eden
} // namespace facebook

45
eden/fs/store/MemoryLocalStore.h Normal file
View File

@ -0,0 +1,45 @@
/*
* Copyright (c) 2018-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
#pragma once
#include <folly/Synchronized.h>
#include <folly/experimental/StringKeyedUnorderedMap.h>
#include "eden/fs/store/LocalStore.h"
namespace facebook {
namespace eden {
/** An implementation of LocalStore that stores values in memory.
* Stored values remain in memory for the lifetime of the
* MemoryLocalStore instance.
* MemoryLocalStore is thread safe, allowing concurrent reads and
* writes from any thread.
* */
class MemoryLocalStore : public LocalStore {
public:
MemoryLocalStore();
void close() override;
StoreResult get(LocalStore::KeySpace keySpace, folly::ByteRange key)
const override;
bool hasKey(LocalStore::KeySpace keySpace, folly::ByteRange key)
const override;
void put(
LocalStore::KeySpace keySpace,
folly::ByteRange key,
folly::ByteRange value) override;
std::unique_ptr<LocalStore::WriteBatch> beginWrite(
size_t bufSize = 0) override;
private:
folly::Synchronized<std::vector<folly::StringKeyedUnorderedMap<std::string>>>
storage_;
};
} // namespace eden
} // namespace facebook

274
eden/fs/store/RocksDbLocalStore.cpp Normal file
View File

@ -0,0 +1,274 @@
/*
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
#include "eden/fs/store/RocksDbLocalStore.h"
#include <folly/Format.h>
#include <folly/Optional.h>
#include <folly/String.h>
#include <folly/experimental/logging/xlog.h>
#include <folly/io/Cursor.h>
#include <folly/io/IOBuf.h>
#include <folly/lang/Bits.h>
#include <rocksdb/db.h>
#include <rocksdb/filter_policy.h>
#include <rocksdb/table.h>
#include <array>
#include "eden/fs/rocksdb/RocksException.h"
#include "eden/fs/rocksdb/RocksHandles.h"
#include "eden/fs/store/StoreResult.h"
using facebook::eden::Hash;
using folly::ByteRange;
using folly::IOBuf;
using folly::Optional;
using folly::StringPiece;
using folly::io::Cursor;
using rocksdb::ReadOptions;
using rocksdb::Slice;
using rocksdb::SliceParts;
using rocksdb::WriteBatch;
using rocksdb::WriteOptions;
using std::string;
using std::unique_ptr;
namespace {
using namespace facebook::eden;
rocksdb::ColumnFamilyOptions makeColumnOptions(uint64_t LRUblockCacheSizeMB) {
rocksdb::ColumnFamilyOptions options;
// We'll never perform range scans on any of the keys that we store.
// This enables bloom filters and a hash policy that improves our
// get/put performance.
options.OptimizeForPointLookup(LRUblockCacheSizeMB);
options.OptimizeLevelStyleCompaction();
return options;
}
/**
* The different key spaces that we desire.
* The ordering is coupled with the values of the LocalStore::KeySpace enum.
*/
const std::vector<rocksdb::ColumnFamilyDescriptor>& columnFamilies() {
// Most of the column families will share the same cache. We
// want the blob data to live in its own smaller cache; the assumption
// is that the vfs cache will compensate for that, together with the
// idea that we shouldn't need to materialize a great many files.
auto options = makeColumnOptions(64);
auto blobOptions = makeColumnOptions(8);
// Meyers singleton to avoid SIOF issues
static const std::vector<rocksdb::ColumnFamilyDescriptor> families{
rocksdb::ColumnFamilyDescriptor{rocksdb::kDefaultColumnFamilyName,
options},
rocksdb::ColumnFamilyDescriptor{"blob", blobOptions},
rocksdb::ColumnFamilyDescriptor{"blobmeta", options},
rocksdb::ColumnFamilyDescriptor{"tree", options},
rocksdb::ColumnFamilyDescriptor{"hgproxyhash", options},
rocksdb::ColumnFamilyDescriptor{"hgcommit2tree", options},
};
return families;
}
rocksdb::Slice _createSlice(folly::ByteRange bytes) {
return Slice(reinterpret_cast<const char*>(bytes.data()), bytes.size());
}
class RocksDbWriteBatch : public LocalStore::WriteBatch {
public:
void put(
LocalStore::KeySpace keySpace,
folly::ByteRange key,
folly::ByteRange value) override;
void put(
LocalStore::KeySpace keySpace,
folly::ByteRange key,
std::vector<folly::ByteRange> valueSlices) override;
void flush() override;
~RocksDbWriteBatch();
// Use LocalStore::beginWrite() to create a write batch
RocksDbWriteBatch(RocksHandles& dbHandles, size_t bufferSize);
void flushIfNeeded();
RocksHandles& dbHandles_;
rocksdb::WriteBatch writeBatch_;
size_t bufSize_;
};
void RocksDbWriteBatch::flush() {
auto pending = writeBatch_.Count();
if (pending == 0) {
return;
}
XLOG(DBG5) << "Flushing " << pending << " entries with data size of "
<< writeBatch_.GetDataSize();
auto status = dbHandles_.db->Write(WriteOptions(), &writeBatch_);
XLOG(DBG5) << "... Flushed";
if (!status.ok()) {
throw RocksException::build(
status, "error putting blob batch in local store");
}
writeBatch_.Clear();
}
void RocksDbWriteBatch::flushIfNeeded() {
auto needFlush = bufSize_ > 0 && writeBatch_.GetDataSize() >= bufSize_;
if (needFlush) {
flush();
}
}
RocksDbWriteBatch::RocksDbWriteBatch(RocksHandles& dbHandles, size_t bufSize)
: LocalStore::WriteBatch(),
dbHandles_(dbHandles),
writeBatch_(bufSize),
bufSize_(bufSize) {}
RocksDbWriteBatch::~RocksDbWriteBatch() {
if (writeBatch_.Count() > 0) {
XLOG(ERR) << "WriteBatch being destroyed with " << writeBatch_.Count()
<< " items pending flush";
}
}
void RocksDbWriteBatch::put(
LocalStore::KeySpace keySpace,
folly::ByteRange key,
folly::ByteRange value) {
writeBatch_.Put(
dbHandles_.columns[keySpace].get(),
_createSlice(key),
_createSlice(value));
flushIfNeeded();
}
void RocksDbWriteBatch::put(
LocalStore::KeySpace keySpace,
folly::ByteRange key,
std::vector<folly::ByteRange> valueSlices) {
std::vector<Slice> slices;
for (auto& valueSlice : valueSlices) {
slices.emplace_back(_createSlice(valueSlice));
}
auto keySlice = _createSlice(key);
SliceParts keyParts(&keySlice, 1);
writeBatch_.Put(
dbHandles_.columns[keySpace].get(),
keyParts,
SliceParts(slices.data(), slices.size()));
flushIfNeeded();
}
} // namespace
namespace facebook {
namespace eden {
RocksDbLocalStore::RocksDbLocalStore(AbsolutePathPiece pathToRocksDb)
: dbHandles_(pathToRocksDb.stringPiece(), columnFamilies()) {}
RocksDbLocalStore::~RocksDbLocalStore() {
#ifdef FOLLY_SANITIZE_ADDRESS
// RocksDB has some race conditions around setting up and tearing down
// the threads that it uses to maintain the database. This manifests
// in our test harness, particularly in a test where we quickly mount
// and then unmount. We see this as an abort with the message:
// "pthread lock: Invalid Argument".
// My assumption is that we're shutting things down before rocks has
// completed initializing. This sleep call is present in the destructor
// to make it more likely that rocks is past that critical point and
// so that we can shutdown successfully.
/* sleep override */ sleep(1);
#endif
}
void RocksDbLocalStore::close() {
dbHandles_.columns.clear();
dbHandles_.db.reset();
}
StoreResult RocksDbLocalStore::get(LocalStore::KeySpace keySpace, ByteRange key)
const {
string value;
auto status = dbHandles_.db.get()->Get(
ReadOptions(),
dbHandles_.columns[keySpace].get(),
_createSlice(key),
&value);
if (!status.ok()) {
if (status.IsNotFound()) {
// Return an empty StoreResult
return StoreResult();
}
// TODO: RocksDB can return a "TryAgain" error.
// Should we try again for the user, rather than re-throwing the error?
// We don't use RocksException::check(), since we don't want to waste our
// time computing the hex string of the key if we succeeded.
throw RocksException::build(
status, "failed to get ", folly::hexlify(key), " from local store");
}
return StoreResult(std::move(value));
}
bool RocksDbLocalStore::hasKey(
LocalStore::KeySpace keySpace,
folly::ByteRange key) const {
string value;
auto status = dbHandles_.db->Get(
ReadOptions(),
dbHandles_.columns[keySpace].get(),
_createSlice(key),
&value);
if (!status.ok()) {
if (status.IsNotFound()) {
return false;
}
// TODO: RocksDB can return a "TryAgain" error.
// Should we try again for the user, rather than re-throwing the error?
// We don't use RocksException::check(), since we don't want to waste our
// time computing the hex string of the key if we succeeded.
throw RocksException::build(
status, "failed to get ", folly::hexlify(key), " from local store");
}
return true;
}
std::unique_ptr<LocalStore::WriteBatch> RocksDbLocalStore::beginWrite(
size_t bufSize) {
return std::make_unique<RocksDbWriteBatch>(dbHandles_, bufSize);
}
void RocksDbLocalStore::put(
LocalStore::KeySpace keySpace,
folly::ByteRange key,
folly::ByteRange value) {
dbHandles_.db->Put(
WriteOptions(),
dbHandles_.columns[keySpace].get(),
_createSlice(key),
_createSlice(value));
}
} // namespace eden
} // namespace facebook

40
eden/fs/store/RocksDbLocalStore.h Normal file
View File

@ -0,0 +1,40 @@
/*
* Copyright (c) 2016-present, Facebook, Inc.
* All rights reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree. An additional grant
* of patent rights can be found in the PATENTS file in the same directory.
*
*/
#pragma once
#include "eden/fs/rocksdb/RocksHandles.h"
#include "eden/fs/store/LocalStore.h"
namespace facebook {
namespace eden {
/** An implementation of LocalStore that uses RocksDB for the underlying
* storage.
*/
class RocksDbLocalStore : public LocalStore {
public:
explicit RocksDbLocalStore(AbsolutePathPiece pathToRocksDb);
~RocksDbLocalStore();
void close() override;
StoreResult get(LocalStore::KeySpace keySpace, folly::ByteRange key)
const override;
bool hasKey(LocalStore::KeySpace keySpace, folly::ByteRange key)
const override;
void put(
LocalStore::KeySpace keySpace,
folly::ByteRange key,
folly::ByteRange value) override;
std::unique_ptr<WriteBatch> beginWrite(size_t bufSize = 0) override;
private:
RocksHandles dbHandles_;
};
} // namespace eden
} // namespace facebook

31
eden/fs/store/hg/HgImporter.cpp
View File

@ -148,7 +148,7 @@ struct HgProxyHash {
static Hash store(
RelativePathPiece path,
Hash hgRevHash,
LocalStore::WriteBatch& writeBatch) {
LocalStore::WriteBatch* writeBatch) {
auto computedPair = prepareToStore(path, hgRevHash);
HgProxyHash::store(computedPair, writeBatch);
return computedPair.first;
@ -182,8 +182,8 @@ struct HgProxyHash {
*/
static void store(
const std::pair<Hash, IOBuf>& computedPair,
LocalStore::WriteBatch& writeBatch) {
writeBatch.put(
LocalStore::WriteBatch* writeBatch) {
writeBatch->put(
KeySpace::HgProxyHashFamily,
computedPair.first,
// Note that this depends on prepareToStore() having called
@ -497,8 +497,8 @@ std::unique_ptr<Tree> HgImporter::importTree(const Hash& id) {
pathInfo.revHash(), // this is really the manifest node
id,
pathInfo.path(),
writeBatch);
writeBatch.flush();
writeBatch.get());
writeBatch->flush();
return tree;
}
@ -506,7 +506,7 @@ std::unique_ptr<Tree> HgImporter::importTreeImpl(
const Hash& manifestNode,
const Hash& edenTreeID,
RelativePathPiece path,
LocalStore::WriteBatch& writeBatch) {
LocalStore::WriteBatch* writeBatch) {
XLOG(DBG6) << "importing tree " << edenTreeID << ": hg manifest "
<< manifestNode << " for path \"" << path << "\"";
@ -516,7 +516,7 @@ std::unique_ptr<Tree> HgImporter::importTreeImpl(
if (path.empty() && manifestNode == kZeroHash) {
auto tree = std::make_unique<Tree>(std::vector<TreeEntry>{}, edenTreeID);
auto serialized = LocalStore::serializeTree(tree.get());
writeBatch.put(
writeBatch->put(
KeySpace::TreeFamily, edenTreeID, serialized.second.coalesce());
return tree;
}
@ -669,7 +669,7 @@ std::unique_ptr<Tree> HgImporter::importTreeImpl(
auto tree = std::make_unique<Tree>(std::move(entries), edenTreeID);
auto serialized = LocalStore::serializeTree(tree.get());
writeBatch.put(
writeBatch->put(
KeySpace::TreeFamily, edenTreeID, serialized.second.coalesce());
return tree;
}
@ -695,11 +695,12 @@ Hash HgImporter::importTreeManifest(StringPiece revName) {
RelativePathPiece path{};
auto proxyInfo = HgProxyHash::prepareToStore(path, manifestNode);
auto writeBatch = store_->beginWrite();
auto tree = importTreeImpl(manifestNode, proxyInfo.first, path, writeBatch);
auto tree =
importTreeImpl(manifestNode, proxyInfo.first, path, writeBatch.get());
// Only write the proxy hash value for this once we've imported
// the root.
HgProxyHash::store(proxyInfo, writeBatch);
writeBatch.flush();
HgProxyHash::store(proxyInfo, writeBatch.get());
writeBatch->flush();
return tree->getHash();
}
@ -713,7 +714,7 @@ Hash HgImporter::importFlatManifest(StringPiece revName) {
Hash HgImporter::importFlatManifest(int fd, LocalStore* store) {
auto writeBatch = store->beginWrite(FLAGS_hgManifestImportBufferSize);
HgManifestImporter importer(store, writeBatch);
HgManifestImporter importer(store, writeBatch.get());
size_t numPaths = 0;
auto start = std::chrono::steady_clock::now();
@ -735,7 +736,7 @@ Hash HgImporter::importFlatManifest(int fd, LocalStore* store) {
// Now process the entries in the chunk
Cursor cursor(&chunkData);
while (!cursor.isAtEnd()) {
readManifestEntry(importer, cursor, writeBatch);
readManifestEntry(importer, cursor, writeBatch.get());
++numPaths;
}
@ -744,7 +745,7 @@ Hash HgImporter::importFlatManifest(int fd, LocalStore* store) {
}
}
writeBatch.flush();
writeBatch->flush();
auto computeEnd = std::chrono::steady_clock::now();
XLOG(DBG2) << "computed trees for " << numPaths << " manifest paths in "
@ -801,7 +802,7 @@ Hash HgImporter::resolveManifestNode(folly::StringPiece revName) {
void HgImporter::readManifestEntry(
HgManifestImporter& importer,
folly::io::Cursor& cursor,
LocalStore::WriteBatch& writeBatch) {
LocalStore::WriteBatch* writeBatch) {
Hash::Storage hashBuf;
cursor.pull(hashBuf.data(), hashBuf.size());
Hash fileRevHash(hashBuf);

4
eden/fs/store/hg/HgImporter.h
View File

@ -208,7 +208,7 @@ class HgImporter {
static void readManifestEntry(
HgManifestImporter& importer,
folly::io::Cursor& cursor,
LocalStore::WriteBatch& writeBatch);
LocalStore::WriteBatch* writeBatch);
/**
* Read a response chunk header from the helper process
*
@ -266,7 +266,7 @@ class HgImporter {
const Hash& manifestNode,
const Hash& edenTreeID,
RelativePathPiece path,
LocalStore::WriteBatch& writeBatch);
LocalStore::WriteBatch* writeBatch);
folly::Subprocess helper_;
const AbsolutePath repoPath_;

10
eden/fs/store/hg/HgManifestImporter.cpp
View File

@ -53,7 +53,7 @@ class HgManifestImporter::PartialTree {
/** Record this node against the store.
* May only be called after compute() has been called (this method
* will check and assert on this). */
Hash record(LocalStore* store, LocalStore::WriteBatch& batch);
Hash record(LocalStore* store, LocalStore::WriteBatch* batch);
/** Compute the serialized version of this tree.
* Records the id and data ready to be stored by a later call
@ -122,7 +122,7 @@ Hash HgManifestImporter::PartialTree::compute(LocalStore* store) {
Hash HgManifestImporter::PartialTree::record(
LocalStore* store,
LocalStore::WriteBatch& batch) {
LocalStore::WriteBatch* batch) {
DCHECK(computed_) << "Must have computed PartialTree prior to recording";
// If the store already has data on this node, then we don't need to
// recurse into any of our children; we're done!
@ -137,7 +137,7 @@ Hash HgManifestImporter::PartialTree::record(
it.record(store, batch);
}
batch.put(LocalStore::KeySpace::TreeFamily, id_, treeData_.coalesce());
batch->put(LocalStore::KeySpace::TreeFamily, id_, treeData_.coalesce());
XLOG(DBG6) << "record tree: '" << path_ << "' --> " << id_.toString() << " ("
<< numPaths_ << " paths, " << trees_.size() << " trees)";
@ -147,7 +147,7 @@ Hash HgManifestImporter::PartialTree::record(
HgManifestImporter::HgManifestImporter(
LocalStore* store,
LocalStore::WriteBatch& writeBatch)
LocalStore::WriteBatch* writeBatch)
: store_(store), writeBatch_(writeBatch) {
// Push the root directory onto the stack
dirStack_.emplace_back(RelativePath(""));
@ -217,7 +217,7 @@ Hash HgManifestImporter::finish() {
dirStack_.pop_back();
CHECK(dirStack_.empty());
writeBatch_.flush();
writeBatch_->flush();
return rootHash;
}

4
eden/fs/store/hg/HgManifestImporter.h
View File

@ -28,7 +28,7 @@ class HgManifestImporter {
public:
explicit HgManifestImporter(
LocalStore* store,
LocalStore::WriteBatch& writeBatch);
LocalStore::WriteBatch* writeBatch);
virtual ~HgManifestImporter();
/**
@ -62,7 +62,7 @@ class HgManifestImporter {
LocalStore* store_{nullptr};
std::vector<PartialTree> dirStack_;
LocalStore::WriteBatch& writeBatch_;
LocalStore::WriteBatch* writeBatch_;
};
} // namespace eden
} // namespace facebook

3
eden/fs/store/hg/test/HgImportTest.cpp
View File

@ -19,6 +19,7 @@
#include "eden/fs/model/Hash.h"
#include "eden/fs/model/Tree.h"
#include "eden/fs/store/LocalStore.h"
#include "eden/fs/store/MemoryLocalStore.h"
#include "eden/fs/store/hg/HgImportPyError.h"
#include "eden/fs/store/hg/HgImporter.h"
#include "eden/fs/testharness/HgRepo.h"
@ -56,7 +57,7 @@ class HgImportTest : public ::testing::Test {
TemporaryDirectory testDir_{"eden_test"};
AbsolutePath testPath_{testDir_.path().string()};
HgRepo repo_{testPath_ + PathComponentPiece{"repo"}};
LocalStore localStore_{testPath_ + PathComponentPiece{"store"}};
MemoryLocalStore localStore_;
};
void HgImportTest::importTest(bool treemanifest) {

4
eden/fs/store/hg/tester.cpp
View File

@ -21,7 +21,7 @@
#include <sysexits.h>
#include "eden/fs/model/Tree.h"
#include "eden/fs/store/LocalStore.h"
#include "eden/fs/store/RocksDbLocalStore.h"
#include "eden/fs/store/hg/HgImporter.h"
#include "eden/fs/store/hg/HgManifestImporter.h"
#include "eden/fs/utils/PathFuncs.h"
@ -191,7 +191,7 @@ int main(int argc, char* argv[]) {
revName = ".";
}
LocalStore store(rocksPath);
RocksDbLocalStore store(rocksPath);
int returnCode = EX_OK;
if (!FLAGS_flat_import_file.empty()) {

26
eden/fs/store/test/LocalStoreTest.cpp
View File

@ -18,6 +18,7 @@
#include "eden/fs/model/Hash.h"
#include "eden/fs/model/Tree.h"
#include "eden/fs/model/TreeEntry.h"
#include "eden/fs/store/MemoryLocalStore.h"
#include "eden/fs/store/StoreResult.h"
using namespace facebook::eden;
@ -33,8 +34,7 @@ class LocalStoreTest : public ::testing::Test {
protected:
void SetUp() override {
testDir_ = std::make_unique<TemporaryDirectory>("eden_test");
auto path = AbsolutePathPiece{testDir_->path().string()};
store_ = std::make_unique<LocalStore>(path);
store_ = std::make_unique<MemoryLocalStore>();
}
void TearDown() override {
@ -159,22 +159,22 @@ TEST_F(LocalStoreTest, testMultipleBlobWriters) {
StringPiece key3_2 = "damage";
auto batch1 = store_->beginWrite(8192);
batch1.put(KeySpace::BlobFamily, key1_1, StringPiece{"hello world1_1"});
batch1.put(KeySpace::BlobFamily, key1_2, StringPiece{"hello world1_2"});
batch1->put(KeySpace::BlobFamily, key1_1, StringPiece{"hello world1_1"});
batch1->put(KeySpace::BlobFamily, key1_2, StringPiece{"hello world1_2"});
auto batch2 = store_->beginWrite(1024);
batch2.put(KeySpace::BlobFamily, key2_1, StringPiece{"hello world2_1"});
batch2.put(KeySpace::BlobFamily, key2_2, StringPiece{"hello world2_2"});
batch2->put(KeySpace::BlobFamily, key2_1, StringPiece{"hello world2_1"});
batch2->put(KeySpace::BlobFamily, key2_2, StringPiece{"hello world2_2"});
auto batch3 = store_->beginWrite();
batch3.put(KeySpace::BlobFamily, key3_1, StringPiece{"hello world3_1"});
batch3.put(KeySpace::BlobFamily, key3_2, StringPiece{"hello world3_2"});
batch3->put(KeySpace::BlobFamily, key3_1, StringPiece{"hello world3_1"});
batch3->put(KeySpace::BlobFamily, key3_2, StringPiece{"hello world3_2"});
batch1.put(KeySpace::BlobFamily, key1_3, StringPiece{"hello world1_3"});
batch1.put(KeySpace::BlobFamily, key1_4, StringPiece{"hello world1_4"});
batch1->put(KeySpace::BlobFamily, key1_3, StringPiece{"hello world1_3"});
batch1->put(KeySpace::BlobFamily, key1_4, StringPiece{"hello world1_4"});
batch1.flush();
batch2.flush();
batch1->flush();
batch2->flush();
auto result1_1 = store_->get(KeySpace::BlobFamily, key1_1);
auto result2_1 = store_->get(KeySpace::BlobFamily, key2_1);
@ -183,7 +183,7 @@ TEST_F(LocalStoreTest, testMultipleBlobWriters) {
EXPECT_FALSE(store_->get(KeySpace::BlobFamily, key3_1).isValid())
<< "key3_1 is not visible until flush";
batch3.flush();
batch3->flush();
auto result3_1 = store_->get(KeySpace::BlobFamily, key3_1);
EXPECT_EQ("hello world3_1", result3_1.piece())
<< "key3_1 visible after flush";

4
eden/fs/testharness/TestMount.cpp
View File

@ -23,6 +23,7 @@
#include "eden/fs/model/Tree.h"
#include "eden/fs/store/BackingStore.h"
#include "eden/fs/store/LocalStore.h"
#include "eden/fs/store/MemoryLocalStore.h"
#include "eden/fs/store/ObjectStore.h"
#include "eden/fs/store/hg/HgManifestImporter.h"
#include "eden/fs/testharness/FakeBackingStore.h"
@ -163,8 +164,7 @@ void TestMount::initTestDirectory() {
config_ = make_unique<ClientConfig>(mountPath, clientDirectory);
// Create localStore_ and backingStore_
localStore_ =
make_shared<LocalStore>(testDirPath + PathComponentPiece("rocksdb"));
localStore_ = make_shared<MemoryLocalStore>();
backingStore_ = make_shared<FakeBackingStore>(localStore_);
}

4
eden/fs/testharness/test/FakeBackingStoreTest.cpp
View File

@ -17,6 +17,7 @@
#include <sys/stat.h>
#include "eden/fs/store/LocalStore.h"
#include "eden/fs/store/MemoryLocalStore.h"
#include "eden/fs/testharness/TestUtil.h"
#include "eden/fs/utils/PathFuncs.h"
@ -29,8 +30,7 @@ class FakeBackingStoreTest : public ::testing::Test {
protected:
void SetUp() override {
testDir_ = std::make_unique<TemporaryDirectory>("eden_test");
auto path = AbsolutePathPiece{testDir_->path().string()};
localStore_ = std::make_shared<LocalStore>(path);
localStore_ = std::make_shared<MemoryLocalStore>();
store_ = std::make_unique<FakeBackingStore>(localStore_);
}