facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-06 14:58:03 +03:00
Code Issues Projects Releases Wiki Activity

completely remove sqlite overlay

Browse Source 
Summary: It has been a year, we can remove this now.

Reviewed By: genevievehelsel

Differential Revision: D35321702

fbshipit-source-id: d078cbd1c8dfb74f573c395c3ea891294de20d72
This commit is contained in:
Zeyi (Rice) Fan 2022-05-03 14:31:45 -07:00 committed by Facebook GitHub Bot
parent 7f7945d720
commit a4714e39dc
6 changed files with 13 additions and 455 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
eden/fs/config/CheckoutConfig.cpp
View File

@ -326,7 +326,12 @@ std::unique_ptr<CheckoutConfig> CheckoutConfig::loadFromClientDirectory(
config->requireUtf8Path_ = requireUtf8Path ? *requireUtf8Path : true;
auto enableTreeOverlay = repository->get_as<bool>(kEnableTreeOverlay.str());
#ifdef _WIN32
// Treeoverlay is default on Windows
config->enableTreeOverlay_ = enableTreeOverlay.value_or(true);
#else
config->enableTreeOverlay_ = enableTreeOverlay.value_or(false);
#endif
#ifdef _WIN32
auto guid = repository->get_as<std::string>(kRepoGuid.str());

2
eden/fs/inodes/CMakeLists.txt
View File

@ -4,7 +4,7 @@
# GNU General Public License version 2.
if(WIN32)
file(GLOB INODES_SRCS "*.cpp" "sqliteoverlay/*.cpp")
file(GLOB INODES_SRCS "*.cpp")
list(
REMOVE_ITEM
INODES_SRCS

10
eden/fs/inodes/Overlay.cpp
View File

@ -53,7 +53,11 @@ std::unique_ptr<IOverlay> makeOverlay(
localDir, TreeOverlayStore::SynchronousMode::Off);
}
#ifdef _WIN32
return std::make_unique<SqliteOverlay>(localDir);
if (overlayType == Overlay::OverlayType::Legacy) {
throw std::runtime_error(
"Legacy overlay type is not supported. Please reclone.");
}
return std::make_unique<TreeOverlay>(localDir);
#else
return std::make_unique<FsOverlay>(localDir);
#endif
@ -211,9 +215,9 @@ void Overlay::initOverlay(
optNextInodeNumber = checker.getNextInodeNumber();
#else
// SqliteOverlay will always return the value of next Inode number, if we
// TreeOverlay will always return the value of next Inode number, if we
// end up here - it's a bug.
EDEN_BUG() << "Sqlite Overlay is null value for NextInodeNumber";
EDEN_BUG() << "Tree Overlay is null value for NextInodeNumber";
#endif
} else {
hadCleanStartup_ = true;

1
eden/fs/inodes/Overlay.h
View File

@ -19,7 +19,6 @@
#include "eden/fs/inodes/InodeNumber.h"
#include "eden/fs/inodes/overlay/OverlayChecker.h"
#include "eden/fs/inodes/overlay/gen-cpp2/overlay_types.h"
#include "eden/fs/inodes/sqliteoverlay/SqliteOverlay.h"
#include "eden/fs/telemetry/StructuredLogger.h"
#include "eden/fs/utils/CaseSensitivity.h"
#include "eden/fs/utils/DirType.h"

312
eden/fs/inodes/sqliteoverlay/SqliteOverlay.cpp
View File

@ -1,312 +0,0 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This software may be used and distributed according to the terms of the
* GNU General Public License version 2.
*/
#include "eden/fs/inodes/sqliteoverlay/SqliteOverlay.h"
#include <folly/File.h>
#include <folly/String.h>
#include <folly/container/Array.h>
#include <folly/logging/xlog.h>
#include <thrift/lib/cpp2/protocol/Serializer.h>
#include <iostream>
#include "eden/fs/inodes/InodeNumber.h"
#include "eden/fs/sqlite/PersistentSqliteStatement.h"
#include "eden/fs/sqlite/SqliteStatement.h"
#include "eden/fs/store/StoreResult.h"
#include "eden/fs/utils/Bug.h"
#include "eden/fs/utils/PathFuncs.h"
namespace facebook {
namespace eden {
using folly::ByteRange;
constexpr folly::StringPiece kInodeTable = "Inode";
constexpr folly::StringPiece kConfigTable = "Config";
constexpr PathComponentPiece kOverlayName{"overlay.db"_pc};
constexpr uint32_t kNextInodeNumber = 1;
constexpr uint64_t kStartInodeNumber = 100;
// TODO: We should customize it by reading the value from the config. This is
// part of the stop gap solution put in place for recovery from unclean
// shutdown. More details in the header file.
constexpr uint64_t kInodeAllocationRange = 100;
struct SqliteOverlay::StatementCache {
explicit StatementCache(SqliteDatabase::Connection& db)
: hasInode{db, "select 1 from ", kInodeTable, " where inode = ?"},
// TODO: we need `or ignore` otherwise we hit primary key violations
// when running our integration tests. This implies that we're
// over-fetching and that we have a perf improvement opportunity.
insertInode{
db,
"insert or replace into ",
kInodeTable,
" values (?,?,?)"},
loadInode{db, "select value from ", kInodeTable, " where inode = ?"},
deleteInode{db, "delete from ", kInodeTable, " where inode = ?"},
writeInodeNumber{
db,
"insert or replace into ",
kConfigTable,
" VALUES(?, ?)"},
readInodeNumber{
db,
"select value from ",
kConfigTable,
" where key = ?"} {}
PersistentSqliteStatement hasInode;
PersistentSqliteStatement insertInode;
PersistentSqliteStatement loadInode;
PersistentSqliteStatement deleteInode;
PersistentSqliteStatement writeInodeNumber;
PersistentSqliteStatement readInodeNumber;
};
SqliteOverlay::SqliteOverlay(AbsolutePathPiece localDir)
: localDir_{std::move(localDir)} {}
SqliteOverlay::~SqliteOverlay() {
cache_.reset();
if (db_) {
db_->close();
}
}
std::optional<InodeNumber> SqliteOverlay::initOverlay(
bool createIfNonExisting) {
if (createIfNonExisting) {
ensureDirectoryExists(localDir_);
}
db_ = std::make_unique<SqliteDatabase>(localDir_ + kOverlayName);
auto db = db_->lock();
// Write ahead log for faster perf https://www.sqlite.org/wal.html
SqliteStatement(db, "PRAGMA journal_mode=WAL").step();
// The Inode table stores the information about each inode. At this point we
// are only using it to store the information about the directory entries
SqliteStatement(
db,
"CREATE TABLE IF NOT EXISTS ",
kInodeTable,
"(",
"inode BIGINT NOT NULL,",
"isdir INT NOT NULL,",
"value BINARY NOT NULL,",
"PRIMARY KEY (inode)",
")")
.step();
SqliteStatement(
db,
"CREATE TABLE IF NOT EXISTS ",
kConfigTable,
"(",
"key INT NOT NULL,",
"value BINARY NOT NULL,",
"PRIMARY KEY (key)",
")")
.step();
cache_ = std::make_unique<StatementCache>(db);
// In the following code we read the last know used inode number and allocate
// a range of inodes by saving the incremented value in db.
uint64_t nextInodeNumber;
auto optNextInodeNumber = readNextInodeNumber(db);
if (optNextInodeNumber.has_value()) {
nextInodeNumber = optNextInodeNumber.value();
} else {
// This will only be true if this is the first run.
nextInodeNumber = kStartInodeNumber;
}
auto nextValue = nextInodeNumber + kStartInodeNumber;
writeNextInodeNumber(db, nextValue);
nextInodeNumber_.store(nextValue, std::memory_order_release);
// The only reason we return an optional value is to have a common interface
// with FsOverlay. This would change once we have implement OverlayChecker.
return std::make_optional(InodeNumber{nextInodeNumber});
}
void SqliteOverlay::close(std::optional<InodeNumber> nextInodeNumber) {
if (nextInodeNumber.has_value()) {
saveNextInodeNumber(nextInodeNumber.value().get());
}
cache_.reset();
db_->close();
}
void SqliteOverlay::updateUsedInodeNumber(uint64_t usedInodeNumber) {
saveNextInodeNumber(usedInodeNumber + 1);
}
std::optional<std::string> SqliteOverlay::load(uint64_t inodeNumber) {
auto db = db_->lock();
auto& stmt = cache_->loadInode.get(db);
// Bind the inode; parameters are 1-based
stmt.bind(1, inodeNumber);
if (stmt.step()) {
// Return the result; columns are 0-based!
auto blob = stmt.columnBlob(0);
return std::string(blob.data(), blob.size());
}
// the inode does not exist
return std::nullopt;
}
bool SqliteOverlay::hasInode(uint64_t inodeNumber) {
auto db = db_->lock();
auto& stmt = cache_->hasInode.get(db);
stmt.bind(1, inodeNumber);
return stmt.step();
}
void SqliteOverlay::save(
uint64_t inodeNumber,
bool isDirectory,
ByteRange value) {
auto db = db_->lock();
auto& stmt = cache_->insertInode.get(db);
const uint32_t dir = isDirectory ? 1 : 0;
stmt.bind(1, inodeNumber);
stmt.bind(2, dir);
stmt.bind(3, value);
stmt.step();
}
void SqliteOverlay::saveOverlayDir(
InodeNumber inodeNumber,
const overlay::OverlayDir& odir) {
// Ask thrift to serialize it.
auto serializedData =
apache::thrift::CompactSerializer::serialize<std::string>(odir);
save(
inodeNumber.getRawValue(),
/*isDirectory=*/true,
folly::StringPiece(serializedData));
}
std::optional<overlay::OverlayDir> SqliteOverlay::loadOverlayDir(
InodeNumber inodeNumber) {
auto serializedData = load(inodeNumber.getRawValue());
if (!serializedData.has_value()) {
return std::nullopt;
}
return apache::thrift::CompactSerializer::deserialize<overlay::OverlayDir>(
serializedData.value());
}
std::optional<overlay::OverlayDir> SqliteOverlay::loadAndRemoveOverlayDir(
InodeNumber inodeNumber) {
auto result = loadOverlayDir(inodeNumber);
removeOverlayData(inodeNumber);
return result;
}
void SqliteOverlay::removeOverlayData(InodeNumber inodeNumber) {
auto db = db_->lock();
auto& stmt = cache_->deleteInode.get(db);
stmt.bind(1, inodeNumber.get());
stmt.step();
}
bool SqliteOverlay::hasOverlayData(InodeNumber inodeNumber) {
return hasInode(inodeNumber.get());
}
void SqliteOverlay::saveNextInodeNumber(uint64_t inodeNumber) {
if (inodeNumber >= nextInodeNumber_.load(std::memory_order_relaxed)) {
auto db = db_->lock();
// Check again in case some other thread won the race to acquire the lock
if (inodeNumber >= nextInodeNumber_.load(std::memory_order_relaxed)) {
auto nextValue = inodeNumber + kInodeAllocationRange;
writeNextInodeNumber(db, nextValue);
nextInodeNumber_.store(nextValue, std::memory_order_relaxed);
}
}
}
std::optional<uint64_t> SqliteOverlay::readNextInodeNumber(
SqliteDatabase::Connection& db) {
auto& stmt = cache_->readInodeNumber.get(db);
// Bind the key; parameters are 1-based
stmt.bind(1, kNextInodeNumber);
if (stmt.step()) {
// Return the result; columns are 0-based!
auto blob = stmt.columnBlob(0);
if (blob.size() != sizeof(uint64_t)) {
throw std::logic_error(fmt::format(
"Unable to fetch the next inode number from the db, size: {}",
blob.size()));
}
uint64_t ino;
std::memcpy(&ino, blob.data(), sizeof(ino));
return std::make_optional(ino);
}
return std::nullopt;
}
void SqliteOverlay::writeNextInodeNumber(
SqliteDatabase::Connection& db,
uint64_t inodeNumber) {
folly::StringPiece ino{ByteRange(
reinterpret_cast<const uint8_t*>(&inodeNumber),
reinterpret_cast<const uint8_t*>(&inodeNumber + 1))};
auto& stmt = cache_->writeInodeNumber.get(db);
stmt.bind(1, kNextInodeNumber);
stmt.bind(2, ino);
stmt.step();
}
#ifndef _WIN32
folly::File SqliteOverlay::createOverlayFile(
InodeNumber /*inodeNumber*/,
folly::ByteRange /*contents*/) {
EDEN_BUG() << "UNIMPLEMENTED";
}
folly::File SqliteOverlay::createOverlayFile(
InodeNumber /*inodeNumber*/,
const folly::IOBuf& /*contents*/) {
EDEN_BUG() << "UNIMPLEMENTED";
}
folly::File SqliteOverlay::openFile(
InodeNumber /*inodeNumber*/,
folly::StringPiece /*headerId*/) {
EDEN_BUG() << "UNIMPLEMENTED";
}
folly::File SqliteOverlay::openFileNoVerify(InodeNumber /*inodeNumber*/) {
EDEN_BUG() << "UNIMPLEMENTED";
}
struct statfs SqliteOverlay::statFs() const {
EDEN_BUG() << "UNIMPLEMENTED";
}
#endif
} // namespace eden
} // namespace facebook

138
eden/fs/inodes/sqliteoverlay/SqliteOverlay.h
View File

@ -1,138 +0,0 @@
/*
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This software may be used and distributed according to the terms of the
* GNU General Public License version 2.
*/
#pragma once
#include <folly/Synchronized.h>
#include "eden/fs/inodes/IOverlay.h"
#include "eden/fs/inodes/overlay/gen-cpp2/overlay_types.h"
#include "eden/fs/sqlite/SqliteDatabase.h"
namespace facebook {
namespace eden {
struct InodeNumber;
/**
* Sqlite overlay stores the directory inode and its entries in the sqlite
* database. This is similar to FsOverlay but doesn't support all the
* functionality. This is only used on Windows right now.
*/
class SqliteOverlay : public IOverlay {
public:
explicit SqliteOverlay(AbsolutePathPiece localDir);
~SqliteOverlay();
SqliteOverlay(const SqliteOverlay&) = delete;
SqliteOverlay& operator=(const SqliteOverlay&) = delete;
SqliteOverlay(SqliteOverlay&&) = delete;
SqliteOverlay&& operator=(SqliteOverlay&&) = delete;
bool supportsSemanticOperations() const override {
return false;
}
/**
* Initialize the overlay, and load the nextInodeNumber. The "close"
* method should be used to release these resources and persist the
* nextInodeNumber.
*
* It ignores the value of createIfNonExisting. The Sqlite
* DB and the tables are created or opened in the contructor and are closed in
* the destructor.
*/
std::optional<InodeNumber> initOverlay(bool createIfNonExisting) override;
/**
* Gracefully, shutdown the overlay, persisting the overlay's
* nextInodeNumber.
*/
void close(std::optional<InodeNumber> nextInodeNumber) override;
const AbsolutePath& getLocalDir() const override {
return localDir_;
}
/**
* Was FsOverlay initialized - i.e., is cleanup (close) necessary.
*/
bool initialized() const override {
return bool(db_);
}
void saveOverlayDir(InodeNumber inodeNumber, const overlay::OverlayDir& odir)
override;
std::optional<overlay::OverlayDir> loadOverlayDir(
InodeNumber inodeNumber) override;
std::optional<overlay::OverlayDir> loadAndRemoveOverlayDir(
InodeNumber inodeNumber) override;
void removeOverlayData(InodeNumber inodeNumber) override;
bool hasOverlayData(InodeNumber inodeNumber) override;
/**
* Update the last used Inode number to a new value. This is a stop gap
* solution for the recovery when Eden doesn't know the last used inode number
* in case of an unclean shutdown.
*
* How it works: The SqliteOverlay allocates a range of inodes and keep
* assigning the inode numbers from that. Once the allocated inode number is
* at the end of range it will allocate a new range. To allocate the range it
* will add the the known value of the last used inode number with the size of
* range and save that value as the last known Inode number. In case of
* unclean shutdown we know that last used inode number must be smaller than
* the Inode number stored in the Sqlite.
*
*/
void updateUsedInodeNumber(uint64_t usedInodeNumber) override;
#ifndef _WIN32
folly::File createOverlayFile(
InodeNumber inodeNumber,
folly::ByteRange contents) override;
folly::File createOverlayFile(
InodeNumber inodeNumber,
const folly::IOBuf& contents) override;
folly::File openFile(InodeNumber inodeNumber, folly::StringPiece headerId)
override;
folly::File openFileNoVerify(InodeNumber inodeNumber) override;
struct statfs statFs() const override;
#endif
private:
struct StatementCache;
std::optional<std::string> load(uint64_t inodeNumber);
bool hasInode(uint64_t inodeNumber);
void save(uint64_t inodeNumber, bool isDirectory, folly::ByteRange value);
// APIs to fetch and save the value of next Inode number
void saveNextInodeNumber(uint64_t inodeNumber);
std::optional<uint64_t> readNextInodeNumber(SqliteDatabase::Connection& db);
void writeNextInodeNumber(
SqliteDatabase::Connection& db,
uint64_t inodeNumber);
// Sqlite db handle
std::unique_ptr<SqliteDatabase> db_;
std::unique_ptr<StatementCache> cache_;
// Path to the folder containing DB.
const AbsolutePath localDir_;
// nextInodeNumber_ is part of a stop gap solution for Windows described
// above. The writes to this are protected by db_ lock.
std::atomic<uint64_t> nextInodeNumber_{0};
}; // namespace eden
} // namespace eden
} // namespace facebook