facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-07 15:27:13 +03:00
Code Issues Projects Releases Wiki Activity

have FileInode read blobs through BlobAccess

Browse Source 
Summary:
FileInode no longer has a strong reference to a blob. Instead, all accesses go through the blob cache. This diff changes the caching behavior for blobs.

The previous behavior was:

When a file's contents are needed in any way, the blob is loaded and referenced by the inode. When the number of open file handles goes to zero, the blob is dropped. The blob is also dropped when the inode is unloaded. Future inode loads or open requests, in that situation, require the blob to be reloaded from the LocalStore.

The new behavior is:

When a file's contents are needed, the blob is loaded and stored into the BlobCache, evicting any if necessary. Future blob requests are satisfied from the BlobCache, pushing it to the back of the eviction queue. When the inode is materialized or unloaded, the blob will be evicted from cache if no other blob has interest in it.

(Note: this ignores all push blocking failures!)

Reviewed By: strager

Differential Revision: D12813912

fbshipit-source-id: 20d20807d2e4a6c37cddab38562fdb7456316aac
This commit is contained in:
Chad Austin 2018-11-28 17:31:38 -08:00 committed by Facebook Github Bot
parent 9bf83c0742
commit 239a4ae8c4
5 changed files with 291 additions and 231 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
eden/fs/inodes/EdenMount.cpp
View File

@ -190,7 +190,7 @@ EdenMount::EdenMount(
dispatcher_{new EdenDispatcher(this)},
objectStore_{std::move(objectStore)},
blobCache_{std::move(blobCache)},
blobAccess_{std::make_unique<BlobAccess>(objectStore_, blobCache_)},
blobAccess_{objectStore_, blobCache_},
overlay_(std::make_unique<Overlay>(config_->getOverlayPath())),
bindMounts_(config_->getBindMounts()),
mountGeneration_(globalProcessGeneration | ++mountGeneration),

8
eden/fs/inodes/EdenMount.h
View File

@ -28,6 +28,7 @@
#include "eden/fs/journal/Journal.h"
#include "eden/fs/model/ParentCommits.h"
#include "eden/fs/service/gen-cpp2/eden_types.h"
#include "eden/fs/store/BlobAccess.h"
#include "eden/fs/takeover/TakeoverData.h"
#include "eden/fs/utils/PathFuncs.h"
@ -43,7 +44,6 @@ namespace facebook {
namespace eden {
class BindMount;
class BlobAccess;
class BlobCache;
class CheckoutConflict;
class ClientConfig;
@ -210,8 +210,8 @@ class EdenMount {
*
* The BlobAccess is guaranteed to be valid for the lifetime of the EdenMount.
*/
BlobAccess* getBlobAccess() const {
return blobAccess_.get();
BlobAccess* getBlobAccess() {
return &blobAccess_;
}
/**
@ -637,7 +637,7 @@ class EdenMount {
std::unique_ptr<EdenDispatcher> dispatcher_;
std::shared_ptr<ObjectStore> objectStore_;
std::shared_ptr<BlobCache> blobCache_;
std::unique_ptr<BlobAccess> blobAccess_;
BlobAccess blobAccess_;
std::unique_ptr<Overlay> overlay_;
InodeNumber dotEdenInodeNumber_{};

426
eden/fs/inodes/FileInode.cpp
View File

@ -23,6 +23,7 @@
#include "eden/fs/inodes/TreeInode.h"
#include "eden/fs/model/Blob.h"
#include "eden/fs/model/Hash.h"
#include "eden/fs/store/BlobAccess.h"
#include "eden/fs/store/BlobMetadata.h"
#include "eden/fs/store/ObjectStore.h"
#include "eden/fs/utils/Bug.h"
@ -224,7 +225,6 @@ void FileInode::LockedState::setMaterialized(folly::File&& file) {
ptr_->file = std::move(file);
ptr_->hash.reset();
ptr_->blob.reset();
ptr_->tag = State::MATERIALIZED_IN_OVERLAY;
ptr_->sha1Valid = false;
}
@ -234,103 +234,142 @@ void FileInode::LockedState::setMaterialized(folly::File&& file) {
* These definitions need to appear before any functions that use them.
********************************************************************/
template <typename Fn>
typename folly::futures::detail::callableResult<FileInode::LockedState, Fn>::
Return
FileInode::runWhileDataLoaded(LockedState state, Fn&& fn) {
auto future = Future<FileHandlePtr>::makeEmpty();
template <typename ReturnType, typename Fn>
ReturnType FileInode::runWhileDataLoaded(
LockedState state,
BlobCache::Interest interest,
std::shared_ptr<const Blob> blob,
Fn&& fn) {
auto future = Future<std::shared_ptr<const Blob>>::makeEmpty();
switch (state->tag) {
case State::BLOB_LOADED:
// We can run the function immediately
return folly::makeFutureWith(
[&] { return std::forward<Fn>(fn)(std::move(state)); });
case State::MATERIALIZED_IN_OVERLAY:
// Open the file, then run the function
state.ensureFileOpen(this);
return folly::makeFutureWith(
[&] { return std::forward<Fn>(fn)(std::move(state)); });
case State::BLOB_NOT_LOADING:
if (!blob) {
// If no blob is given, check if the previous handle is still valid.
blob = state->interestHandle.getBlob();
}
if (!blob) {
// Otherwise, does the cache have one?
//
// The BlobAccess::getBlob call in startLoadingData will also check the
// BlobCache, but by checking it here, we can avoid a transition to
// BLOB_LOADING and back, and also avoid allocating some futures and
// closures.
auto result =
getMount()->getBlobCache()->get(state->hash.value(), interest);
if (result.blob) {
blob = std::move(result.blob);
state->interestHandle = std::move(result.interestHandle);
}
}
if (blob) {
// The blob was still in cache, so we can run the function immediately.
return folly::makeFutureWith([&] {
return std::forward<Fn>(fn)(std::move(state), std::move(blob));
});
} else {
future = startLoadingData(std::move(state), interest);
}
break;
case State::BLOB_LOADING:
// If we're already loading, latch on to the in-progress load
future = state->blobLoadingPromise->getFuture();
state.unlock();
break;
case State::NOT_LOADED:
future = startLoadingData(std::move(state));
break;
case State::MATERIALIZED_IN_OVERLAY:
state.ensureFileOpen(this);
return folly::makeFutureWith(
[&] { return std::forward<Fn>(fn)(std::move(state), nullptr); });
}
return std::move(future).thenValue([self = inodePtrFromThis(),
fn = std::forward<Fn>(fn)](
FileHandlePtr /* handle */) mutable {
// Simply call runWhileDataLoaded() again when we we finish loading the blob
// data. The state should be BLOB_LOADED or MATERIALIZED_IN_OVERLAY this
// time around.
auto stateLock = LockedState{self};
DCHECK(
stateLock->tag == State::BLOB_LOADED ||
stateLock->tag == State::MATERIALIZED_IN_OVERLAY)
<< "unexpected FileInode state after loading: " << stateLock->tag;
return self->runWhileDataLoaded(std::move(stateLock), std::forward<Fn>(fn));
});
return std::move(future).thenValue(
[self = inodePtrFromThis(), fn = std::forward<Fn>(fn), interest](
std::shared_ptr<const Blob> blob) mutable {
// Simply call runWhileDataLoaded() again when we we finish loading the
// blob data. The state should be BLOB_NOT_LOADING or
// MATERIALIZED_IN_OVERLAY this time around.
auto stateLock = LockedState{self};
DCHECK(
stateLock->tag == State::BLOB_NOT_LOADING ||
stateLock->tag == State::MATERIALIZED_IN_OVERLAY)
<< "unexpected FileInode state after loading: " << stateLock->tag;
return self->runWhileDataLoaded<ReturnType>(
std::move(stateLock),
interest,
std::move(blob),
std::forward<Fn>(fn));
});
}
template <typename Fn>
typename folly::futures::detail::callableResult<FileInode::LockedState, Fn>::
Return
FileInode::runWhileMaterialized(LockedState state, Fn&& fn) {
auto future = Future<FileHandlePtr>::makeEmpty();
FileInode::runWhileMaterialized(
LockedState state,
std::shared_ptr<const Blob> blob,
Fn&& fn) {
auto future = Future<std::shared_ptr<const Blob>>::makeEmpty();
switch (state->tag) {
case State::BLOB_LOADED: {
// We have the blob data loaded.
// Materialize the file now.
materializeNow(state);
// Call materializeInParent before we return, after we are
// sure the state lock has been released. This does mean that our parent
// won't have updated our state until after the caller's function runs,
// but this is okay. There is always a brief gap between when we
// materialize ourself and when our parent gets updated to indicate this.
// If we do crash during this period it is not too unreasonable that
// recent change right before the crash might be reverted to their
// non-materialized state.
SCOPE_EXIT {
CHECK(state.isNull());
materializeInParent();
};
// Note that we explicitly create a temporary LockedState object
// to pass to the caller to ensure that the state lock will be released
// when they return, even if the caller's function accepts the state as
// an rvalue-reference and does not release it themselves.
return folly::makeFutureWith(
[&] { return std::forward<Fn>(fn)(LockedState{std::move(state)}); });
}
case State::MATERIALIZED_IN_OVERLAY:
// Open the file, then run the function
state.ensureFileOpen(this);
return folly::makeFutureWith(
[&] { return std::forward<Fn>(fn)(LockedState{std::move(state)}); });
case State::BLOB_NOT_LOADING:
if (!blob) {
// If a blob wasn't passed into runWhileMaterialized, try to see if the
// last one we loaded is still alive.
blob = state->interestHandle.getBlob();
}
if (blob) {
// We have the blob data loaded.
// Materialize the file now.
materializeNow(state, blob);
// Call materializeInParent before we return, after we are
// sure the state lock has been released. This does mean that our
// parent won't have updated our state until after the caller's function
// runs, but this is okay. There is always a brief gap between when we
// materialize ourself and when our parent gets updated to indicate
// this. If we do crash during this period it is not too unreasonable
// that recent change right before the crash might be reverted to their
// non-materialized state.
SCOPE_EXIT {
CHECK(state.isNull());
materializeInParent();
};
// Note that we explicitly create a temporary LockedState object
// to pass to the caller to ensure that the state lock will be released
// when they return, even if the caller's function accepts the state as
// an rvalue-reference and does not release it themselves.
return folly::makeFutureWith([&] {
return std::forward<Fn>(fn)(LockedState{std::move(state)});
});
}
// The blob must be loaded, so kick that off. There's no point in caching
// it in memory - the blob will immediately be written into the overlay
// and then dropped.
future = startLoadingData(
std::move(state), BlobCache::Interest::UnlikelyNeededAgain);
break;
case State::BLOB_LOADING:
// If we're already loading, latch on to the in-progress load
future = state->blobLoadingPromise->getFuture();
state.unlock();
break;
case State::NOT_LOADED:
future = startLoadingData(std::move(state));
break;
case State::MATERIALIZED_IN_OVERLAY:
// Open the file, then run the function
state.ensureFileOpen(this);
return folly::makeFutureWith(
[&] { return std::forward<Fn>(fn)(LockedState{std::move(state)}); });
}
return std::move(future).thenValue(
[self = inodePtrFromThis(),
fn = std::forward<Fn>(fn)](FileHandlePtr /* handle */) mutable {
// Simply call runWhileDataLoaded() again when we we finish loading the
// blob data. The state should be BLOB_LOADED or
// MATERIALIZED_IN_OVERLAY this time around.
fn = std::forward<Fn>(fn)](std::shared_ptr<const Blob> blob) mutable {
// Simply call runWhileMaterialized() again when we we are finished
// loading the blob data.
auto stateLock = LockedState{self};
DCHECK(
stateLock->tag == State::BLOB_LOADED ||
stateLock->tag == State::BLOB_NOT_LOADING ||
stateLock->tag == State::MATERIALIZED_IN_OVERLAY)
<< "unexpected FileInode state after loading: " << stateLock->tag;
return self->runWhileMaterialized(
std::move(stateLock), std::forward<Fn>(fn));
std::move(stateLock), std::move(blob), std::forward<Fn>(fn));
});
}
@ -339,8 +378,7 @@ typename std::result_of<Fn(FileInode::LockedState&&)>::type
FileInode::truncateAndRun(LockedState state, Fn&& fn) {
auto future = Future<FileHandlePtr>::makeEmpty();
switch (state->tag) {
case State::NOT_LOADED:
case State::BLOB_LOADED:
case State::BLOB_NOT_LOADING:
case State::BLOB_LOADING: {
// We are not materialized yet. We need to materialize the file now.
//
@ -354,51 +392,37 @@ FileInode::truncateAndRun(LockedState state, Fn&& fn) {
// materialized in our parent TreeInode.
// - If we successfully materialized the file and were in the
// BLOB_LOADING state, fulfill the blobLoadingPromise.
std::shared_ptr<EdenFileHandle> handle;
std::optional<folly::SharedPromise<FileHandlePtr>> loadingPromise;
std::optional<folly::SharedPromise<std::shared_ptr<const Blob>>>
loadingPromise;
SCOPE_EXIT {
if (loadingPromise) {
loadingPromise->setValue(std::move(handle));
// If transitioning from the loading state to materialized, fulfill
// the loading promise will null. Callbacks will have to handle the
// case that the state is now materialized.
loadingPromise->setValue(nullptr);
}
};
// If we are currently in the BLOB_LOADING state, we first need to create
// an EdenFileHandle object to use to fulfill the blobLoadingPromise.
// We do this early on so that we cannot fail to create the file handle
// after we have successfully materialized the file.
//
// We move the LockedState object into an inner scope when we do this, to
// ensure that the LockedState always gets destroyed before the
// EdenFileHandle object. The EdenFileHandle destructor requires
// acquiring the state lock itself, so the lock cannot still be held when
// the EdenFileHandle destructor runs.
{
LockedState innerState(std::move(state));
if (innerState->tag == State::BLOB_LOADING) {
innerState.createHandleInOuterScope(inodePtrFromThis(), &handle);
}
// Call materializeAndTruncate()
materializeAndTruncate(state);
// Call materializeAndTruncate()
materializeAndTruncate(innerState);
// Now that materializeAndTruncate() has succeeded, extract the
// blobLoadingPromise so we can fulfill it as we exit.
loadingPromise = std::move(state->blobLoadingPromise);
state->blobLoadingPromise.reset();
// Also call materializeInParent() as we exit, before fulfilling the
// blobLoadingPromise.
SCOPE_EXIT {
CHECK(state.isNull());
materializeInParent();
};
// Now that materializeAndTruncate() has succeeded, extract the
// blobLoadingPromise so we can fulfill it as we exit.
loadingPromise = std::move(innerState->blobLoadingPromise);
innerState->blobLoadingPromise.reset();
// Also call materializeInParent() as we exit, before fulfilling the
// blobLoadingPromise.
SCOPE_EXIT {
CHECK(innerState.isNull());
materializeInParent();
};
// Now invoke the input function.
// Note that we explicitly create a temporary LockedState object
// to pass to the caller to ensure that the state lock will be released
// when they return, even if the caller's function accepts the state as
// an rvalue-reference and does not release it themselves.
return std::forward<Fn>(fn)(LockedState{std::move(innerState)});
}
// Now invoke the input function.
// Note that we explicitly create a temporary LockedState object
// to pass to the caller to ensure that the state lock will be released
// when they return, even if the caller's function accepts the state as
// an rvalue-reference and does not release it themselves.
return std::forward<Fn>(fn)(LockedState{std::move(state)});
}
case State::MATERIALIZED_IN_OVERLAY:
// We are already materialized.
@ -415,7 +439,7 @@ FileInode::truncateAndRun(LockedState state, Fn&& fn) {
********************************************************************/
FileInodeState::FileInodeState(const std::optional<Hash>& h) : hash(h) {
tag = hash ? NOT_LOADED : MATERIALIZED_IN_OVERLAY;
tag = hash ? BLOB_NOT_LOADING : MATERIALIZED_IN_OVERLAY;
checkInvariants();
}
@ -432,33 +456,22 @@ FileInodeState::~FileInodeState() = default;
void FileInodeState::checkInvariants() {
switch (tag) {
case NOT_LOADED:
case BLOB_NOT_LOADING:
CHECK(hash);
CHECK(!blobLoadingPromise);
CHECK(!blob);
CHECK(!file);
CHECK(!sha1Valid);
return;
case BLOB_LOADING:
CHECK(hash);
CHECK(blobLoadingPromise);
CHECK(!blob);
CHECK(!file);
CHECK(!sha1Valid);
return;
case BLOB_LOADED:
CHECK(hash);
CHECK(!blobLoadingPromise);
CHECK(blob);
CHECK(!file);
CHECK(!sha1Valid);
DCHECK_EQ(blob->getHash(), hash.value());
return;
case MATERIALIZED_IN_OVERLAY:
// 'materialized'
CHECK(!hash);
CHECK(!blobLoadingPromise);
CHECK(!blob);
if (file) {
CHECK_GT(openCount, 0);
}
@ -482,19 +495,12 @@ void FileInodeState::decOpenCount() {
--openCount;
if (openCount == 0) {
switch (tag) {
case BLOB_LOADED:
blob.reset();
tag = NOT_LOADED;
case BLOB_NOT_LOADING:
case BLOB_LOADING:
break;
case MATERIALIZED_IN_OVERLAY:
// TODO: Before closing the file handle, it might make sense to write
// in-memory timestamps into the overlay, even if the inode remains in
// memory. This would ensure timestamps persist even if the edenfs
// process crashes or otherwise exits without unloading all inodes.
file.close();
break;
default:
break;
}
}
}
@ -606,7 +612,7 @@ folly::Future<Dispatcher::Attr> FileInode::setattr(
if (truncate) {
return truncateAndRun(std::move(state), setAttrs);
} else {
return runWhileMaterialized(std::move(state), setAttrs);
return runWhileMaterialized(std::move(state), nullptr, setAttrs);
}
}
@ -725,7 +731,7 @@ folly::Future<std::shared_ptr<FileHandle>> FileInode::open(int flags) {
// Since we just want to materialize the file and don't need to do
// anything else we pass in a no-op lambda function.
(void)runWhileMaterialized(
std::move(state), [](LockedState&&) { return 0; });
std::move(state), nullptr, [](LockedState&&) { return 0; });
}
}
@ -763,10 +769,9 @@ Future<Hash> FileInode::getSha1() {
auto state = LockedState{this};
switch (state->tag) {
case State::NOT_LOADED:
case State::BLOB_NOT_LOADING:
case State::BLOB_LOADING:
case State::BLOB_LOADED:
// If a file is not materialized it should have a hash value.
// If a file is not materialized, it should have a hash value.
return getObjectStore()->getSha1(state->hash.value());
case State::MATERIALIZED_IN_OVERLAY:
state.ensureFileOpen(this);
@ -794,9 +799,8 @@ folly::Future<struct stat> FileInode::stat() {
getMetadataLocked(*state).applyToStat(st);
switch (state->tag) {
case State::NOT_LOADED:
case State::BLOB_NOT_LOADING:
case State::BLOB_LOADING:
case State::BLOB_LOADED:
CHECK(state->hash.has_value());
// While getBlobMetadata will sometimes need to fetch a blob to compute
// the size and SHA-1, if it's already known, use the cached metadata to
@ -854,13 +858,31 @@ void FileInode::fsync(bool datasync) {
checkUnixError(res);
}
Future<string> FileInode::readAll(CacheHint /*cacheHint*/) {
return runWhileDataLoaded(
Future<string> FileInode::readAll(CacheHint cacheHint) {
auto interest = BlobCache::Interest::LikelyNeededAgain;
switch (cacheHint) {
case CacheHint::NotNeededAgain:
interest = BlobCache::Interest::UnlikelyNeededAgain;
break;
case CacheHint::LikelyNeededAgain:
// readAll() with LikelyNeededAgain is primarily called for files read
// by Eden itself, like .gitignore, and for symlinks on kernels that don't
// cache readlink. At least keep the blob around while the inode is
// loaded.
interest = BlobCache::Interest::WantHandle;
break;
}
return runWhileDataLoaded<Future<string>>(
LockedState{this},
[self = inodePtrFromThis()](LockedState&& state) -> Future<string> {
interest,
nullptr,
[self = inodePtrFromThis()](
LockedState&& state, std::shared_ptr<const Blob> blob) -> string {
std::string result;
switch (state->tag) {
case State::MATERIALIZED_IN_OVERLAY: {
DCHECK(!blob);
// Note that this code requires a write lock on state_ because the
// lseek() call modifies the file offset of the file descriptor.
auto rc = lseek(state->file.fd(), Overlay::kHeaderLength, SEEK_SET);
@ -869,8 +891,8 @@ Future<string> FileInode::readAll(CacheHint /*cacheHint*/) {
folly::readFile(state->file.fd(), result);
break;
}
case State::BLOB_LOADED: {
const auto& contentsBuf = state->blob->getContents();
case State::BLOB_NOT_LOADING: {
const auto& contentsBuf = blob->getContents();
folly::io::Cursor cursor(&contentsBuf);
result =
cursor.readFixedString(contentsBuf.computeChainDataLength());
@ -888,14 +910,22 @@ Future<string> FileInode::readAll(CacheHint /*cacheHint*/) {
}
Future<BufVec> FileInode::read(size_t size, off_t off) {
return runWhileDataLoaded(
return runWhileDataLoaded<Future<BufVec>>(
LockedState{this},
[size, off, self = inodePtrFromThis()](LockedState&& state) {
BlobCache::Interest::WantHandle,
nullptr,
[size, off, self = inodePtrFromThis()](
LockedState&& state, std::shared_ptr<const Blob> blob) -> BufVec {
SCOPE_SUCCESS {
self->updateAtimeLocked(*state);
};
// TODO: if blob, add [size, off] to coverage set and if coverage is
// full, drop interest.
// Materialized either before or during blob load.
if (state->tag == State::MATERIALIZED_IN_OVERLAY) {
state.ensureFileOpen(self.get());
auto buf = folly::IOBuf::createCombined(size);
auto res = ::pread(
state->file.fd(),
@ -906,25 +936,26 @@ Future<BufVec> FileInode::read(size_t size, off_t off) {
checkUnixError(res);
buf->append(res);
return BufVec{std::move(buf)};
} else {
// runWhileDataLoaded() ensures that the state is either
// MATERIALIZED_IN_OVERLAY or BLOB_LOADED
DCHECK_EQ(state->tag, State::BLOB_LOADED);
auto buf = state->blob->getContents();
folly::io::Cursor cursor(&buf);
if (!cursor.canAdvance(off)) {
// Seek beyond EOF. Return an empty result.
return BufVec{folly::IOBuf::wrapBuffer("", 0)};
}
cursor.skip(off);
std::unique_ptr<folly::IOBuf> result;
cursor.cloneAtMost(result, size);
return BufVec{std::move(result)};
}
// runWhileDataLoaded() ensures that the state is either
// MATERIALIZED_IN_OVERLAY or BLOB_NOT_LOADING
DCHECK_EQ(state->tag, State::BLOB_NOT_LOADING);
DCHECK(blob) << "blob missing after load completed";
auto buf = blob->getContents();
folly::io::Cursor cursor(&buf);
if (!cursor.canAdvance(off)) {
// Seek beyond EOF. Return an empty result.
return BufVec{folly::IOBuf::wrapBuffer("", 0)};
}
cursor.skip(off);
std::unique_ptr<folly::IOBuf> result;
cursor.cloneAtMost(result, size);
return BufVec{std::move(result)};
});
}
@ -957,6 +988,7 @@ size_t FileInode::writeImpl(
folly::Future<size_t> FileInode::write(BufVec&& buf, off_t off) {
return runWhileMaterialized(
LockedState{this},
nullptr,
[buf = std::move(buf), off, self = inodePtrFromThis()](
LockedState&& state) {
auto vec = buf.getIov();
@ -977,6 +1009,7 @@ folly::Future<size_t> FileInode::write(folly::StringPiece data, off_t off) {
return runWhileMaterialized(
std::move(state),
nullptr,
[data = data.str(), off, self = inodePtrFromThis()](
LockedState&& stateLock) {
struct iovec iov;
@ -986,14 +1019,16 @@ folly::Future<size_t> FileInode::write(folly::StringPiece data, off_t off) {
});
}
Future<FileInode::FileHandlePtr> FileInode::startLoadingData(
LockedState state) {
DCHECK_EQ(state->tag, State::NOT_LOADED);
Future<std::shared_ptr<const Blob>> FileInode::startLoadingData(
LockedState state,
BlobCache::Interest interest) {
DCHECK_EQ(state->tag, State::BLOB_NOT_LOADING);
// Start the blob load first in case this throws an exception.
// Ideally the state transition is no-except in tandem with the
// Future's .then call.
auto blobFuture = getObjectStore()->getBlob(state->hash.value());
auto getBlobFuture =
getMount()->getBlobAccess()->getBlob(state->hash.value(), interest);
// Everything from here through blobFuture.then should be noexcept.
state->blobLoadingPromise.emplace();
@ -1004,11 +1039,16 @@ Future<FileInode::FileHandlePtr> FileInode::startLoadingData(
state.unlock();
auto self = inodePtrFromThis(); // separate line for formatting
std::move(blobFuture)
.thenTry([self](folly::Try<std::shared_ptr<const Blob>> tryBlob) mutable {
std::move(getBlobFuture)
.thenTry([self](folly::Try<BlobCache::GetResult> tryResult) mutable {
auto state = LockedState{self};
switch (state->tag) {
case State::BLOB_NOT_LOADING:
EDEN_BUG()
<< "A blob load finished when the inode was in BLOB_NOT_LOADING state";
return;
// Since the load doesn't hold the state lock for its duration,
// sanity check that the inode is still in loading state.
//
@ -1018,32 +1058,28 @@ Future<FileInode::FileHandlePtr> FileInode::startLoadingData(
case State::BLOB_LOADING: {
auto promise = std::move(*state->blobLoadingPromise);
state->blobLoadingPromise.reset();
state->tag = State::BLOB_NOT_LOADING;
if (tryBlob.hasValue()) {
// Transition to 'loaded' state.
state.incOpenCount();
state->blob = std::move(tryBlob.value());
state->tag = State::BLOB_LOADED;
promise.setValue(state.unlockAndCreateHandle(std::move(self)));
} else {
state->tag = State::NOT_LOADED;
// Call the Future's subscribers while the state_ lock is not
// held. Even if the FileInode has transitioned to a materialized
// state, any pending loads must be unblocked.
// Call the Future's subscribers while the state_ lock is not
// held. Even if the FileInode has transitioned to a materialized
// state, any pending loads must be unblocked.
if (tryResult.hasValue()) {
state->interestHandle = std::move(tryResult->interestHandle);
state.unlock();
promise.setException(tryBlob.exception());
promise.setValue(std::move(tryResult->blob));
} else {
state.unlock();
promise.setException(std::move(tryResult).exception());
}
break;
return;
}
case State::MATERIALIZED_IN_OVERLAY:
// The load raced with a someone materializing the file to truncate
// it. Nothing left to do here.
break;
default:
EDEN_BUG()
<< "Inode left in unexpected state after getBlob() completed";
// it. Nothing left to do here. The truncation completed the
// promise with a null blob.
CHECK_EQ(false, state->blobLoadingPromise.has_value());
return;
}
})
.thenError([](folly::exception_wrapper&&) {
@ -1059,10 +1095,11 @@ Future<FileInode::FileHandlePtr> FileInode::startLoadingData(
return resultFuture;
}
void FileInode::materializeNow(LockedState& state) {
// This function should only be called from the BLOB_LOADED state
DCHECK_EQ(state->tag, State::BLOB_LOADED);
CHECK(state->blob);
void FileInode::materializeNow(
LockedState& state,
std::shared_ptr<const Blob> blob) {
// This function should only be called from the BLOB_NOT_LOADING state
DCHECK_EQ(state->tag, State::BLOB_NOT_LOADING);
// Look up the blob metadata so we can get the blob contents SHA1
// Since this uses state->hash we perform this before calling
@ -1070,7 +1107,7 @@ void FileInode::materializeNow(LockedState& state) {
auto blobSha1 = getObjectStore()->getSha1(state->hash.value());
auto file = getMount()->getOverlay()->createOverlayFile(
getNodeId(), state->blob->getContents());
getNodeId(), blob->getContents());
state.setMaterialized(std::move(file));
// If we have a SHA-1 from the metadata, apply it to the new file. This
@ -1099,7 +1136,6 @@ void FileInode::materializeAndTruncate(LockedState& state) {
void FileInode::truncateInOverlay(LockedState& state) {
CHECK_EQ(state->tag, State::MATERIALIZED_IN_OVERLAY);
CHECK(!state->hash);
CHECK(!state->blob);
state.ensureFileOpen(this);
checkUnixError(ftruncate(state->file.fd(), 0 + Overlay::kHeaderLength));

81
eden/fs/inodes/FileInode.h
View File

@ -17,6 +17,7 @@
#include "eden/fs/inodes/CacheHint.h"
#include "eden/fs/inodes/InodeBase.h"
#include "eden/fs/model/Tree.h"
#include "eden/fs/store/BlobCache.h"
namespace folly {
class File;
@ -39,25 +40,24 @@ class ObjectStore;
* any member variables of FileInode.
*
* A FileInode can be in one of three states:
* - not loaded
* - not loading: the blob may be in cache, but it is not currently being
* loaded
* - loading: fetching data from backing store, but it's not available yet
* - loaded: contents has been imported from mercurial and is accessible
* - materialized: contents are written into overlay and file handle is open
*
* Valid state transitions:
* - not loaded -> loading
* - not loaded -> materialized (O_TRUNC)
* - loading -> loaded
* - loading -> materialized (O_TRUNC)
* - loaded -> materialized
* - not loading -> loading
* - not loading -> materialized (O_TRUNC)
* - loading -> not loaded (blob available during transition)
* - loading -> materialized (O_TRUNC or not)
* - loading -> not loading -> materialized
*/
struct FileInodeState {
using FileHandlePtr = std::shared_ptr<EdenFileHandle>;
enum Tag : uint8_t {
NOT_LOADED,
BLOB_NOT_LOADING,
BLOB_LOADING,
BLOB_LOADED,
MATERIALIZED_IN_OVERLAY,
};
@ -86,8 +86,8 @@ struct FileInodeState {
}
/**
* Decrement the openCount, releasing the blob or file if the open count is
* now zero.
* Decrement the openCount, closing any open overlay file handles if the open
* count is now zero.
*/
void decOpenCount();
@ -99,20 +99,33 @@ struct FileInodeState {
Tag tag;
/**
* Set only in 'not loaded', 'loading', and 'loaded' states, none otherwise.
* TODO: Perhaps we ought to simply leave this defined...
* Set only in 'not loading' and 'loading' states. std::nullopt otherwise.
*/
std::optional<Hash> hash;
/**
* Set if 'loading'.
* Set if 'loading'. Unset when load completes.
*
* It's possible for this future to complete with a null blob - that happens
* if a truncate operation occurs during load. In that case, the future is
* completed and the inode transitions to the materialized state without
* a blob. Callbacks on this future must handle that case.
*/
std::optional<folly::SharedPromise<FileHandlePtr>> blobLoadingPromise;
std::optional<folly::SharedPromise<std::shared_ptr<const Blob>>>
blobLoadingPromise;
/**
* Set if 'loaded', references immutable data from the backing store.
* If the blob has ever been loaded from cache, this handle represents this
* inode's interest in it. By explicitly resetting the interest handle, the
* inode indicates to the cache that the blob can be released.
*
* This also indicates to the cache that the blob is no longer needed in
* memory when the FileInode is deallocated.
*
* Before attempting to reload the blob, check if the interestHandle has it
* first.
*/
std::shared_ptr<const Blob> blob;
BlobInterestHandle interestHandle;
/**
* If backed by an overlay file, whether the sha1 xattr is valid
@ -274,15 +287,21 @@ class FileInode final : public InodeBaseMetadata<FileInodeState> {
/**
* Run a function with the FileInode data loaded.
*
* fn(state) will be invoked when state->tag is either BLOB_LOADED or
* MATERIALIZED_IN_OVERLAY. If state->tag is MATERIALIZED_IN_OVERLAY,
* state->file will be available.
* fn(state, blob) will be invoked when state->tag is either NOT_LOADING or
* MATERIALIZED_IN_OVERLAY. If state->tag is MATERIALIZED_IN_OVERLAY,
* state->file will be available. If state->tag is NOT_LOADING, then the
* second argument will be a non-null std::shared_ptr<const Blob>.
*
* Returns a Future with the result of fn(state_.wlock())
* The blob parameter is used when recursing.
*
* Returns a Future with the result of fn(state_.wlock(), blob)
*/
template <typename Fn>
typename folly::futures::detail::callableResult<LockedState, Fn>::Return
runWhileDataLoaded(LockedState state, Fn&& fn);
template <typename ReturnType, typename Fn>
ReturnType runWhileDataLoaded(
LockedState state,
BlobCache::Interest interest,
std::shared_ptr<const Blob> blob,
Fn&& fn);
/**
* Run a function with the FileInode materialized.
@ -294,7 +313,10 @@ class FileInode final : public InodeBaseMetadata<FileInodeState> {
*/
template <typename Fn>
typename folly::futures::detail::callableResult<LockedState, Fn>::Return
runWhileMaterialized(LockedState state, Fn&& fn);
runWhileMaterialized(
LockedState state,
std::shared_ptr<const Blob> blob,
Fn&& fn);
/**
* Truncate the file and then call a function.
@ -320,8 +342,9 @@ class FileInode final : public InodeBaseMetadata<FileInodeState> {
* runWhileMaterialized(). Most other callers should use
* runWhileDataLoaded() or runWhileMaterialized() instead.
*/
FOLLY_NODISCARD folly::Future<FileHandlePtr> startLoadingData(
LockedState state);
FOLLY_NODISCARD folly::Future<std::shared_ptr<const Blob>> startLoadingData(
LockedState state,
BlobCache::Interest interest);
/**
* Materialize the file as an empty file in the overlay.
@ -354,13 +377,13 @@ class FileInode final : public InodeBaseMetadata<FileInodeState> {
void truncateInOverlay(LockedState& state);
/**
* Transition from BLOB_LOADED to MATERIALIZED_IN_OVERLAY by copying the
* Transition from NOT_LOADING to MATERIALIZED_IN_OVERLAY by copying the
* blob into the overlay.
*
* The input LockedState object will be updated to hold an open refcount,
* and state->file will be valid when this function returns.
*/
void materializeNow(LockedState& state);
void materializeNow(LockedState& state, std::shared_ptr<const Blob> blob);
/**
* Get a FileInodePtr to ourself.

5
eden/fs/testharness/TestMount.cpp
View File

@ -78,8 +78,9 @@ bool TestMountFile::operator==(const TestMountFile& other) const {
}
TestMount::TestMount()
: blobCache_(
BlobCache::create(kBlobCacheMaximumSize, kBlobCacheMinimumEntries)),
: blobCache_{BlobCache::create(
kBlobCacheMaximumSize,
kBlobCacheMinimumEntries)},
privHelper_{make_shared<FakePrivHelper>()},
serverExecutor_{make_shared<folly::ManualExecutor>()} {
// Initialize the temporary directory.