facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-05 06:18:07 +03:00
Code Issues Projects Releases Wiki Activity

get NfsDispatcherImpl building on Windows

Browse Source 
Summary:
I'm getting nfs to build on windows to prototype it and see how feasible it
might be as an option on Windows. PrjFS has a very different model than EdenFS,
and that has made EdenFS correctness on Windows very difficult. NFS may be
easier to get correct, though the performance is suspect. Just exploring
options here.

After this change NFS builds on Windows!

NOTE: this one is more than removing ifdefs, probably the most non trivial in the stack, please review carefully

Reviewed By: xavierd

Differential Revision: D44153443

fbshipit-source-id: b07e19f8bde1aa455eec97647ea450849133041f
This commit is contained in:
Katie Mancini 2023-04-06 13:00:32 -07:00 committed by Facebook GitHub Bot
parent 30004e3b68
commit 04c2b8b54c
8 changed files with 195 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

176
eden/fs/inodes/FileInode.cpp
View File

@ -34,6 +34,7 @@
#include "eden/fs/utils/FileHash.h"
#include "eden/fs/utils/FileUtils.h"
#include "eden/fs/utils/ImmediateFuture.h"
#include "eden/fs/utils/NotImplemented.h"
#include "eden/fs/utils/PathFuncs.h"
#include "eden/fs/utils/UnboundedQueueExecutor.h"
#include "eden/fs/utils/XAttr.h"
@ -486,10 +487,10 @@ FileInode::FileInode(
: Base(ino, initialMode, initialTimestamps, std::move(parentInode), name),
state_(folly::in_place) {}
#ifndef _WIN32
ImmediateFuture<struct stat> FileInode::setattr(
const DesiredMetadata& desired,
const ObjectFetchContextPtr& fetchContext) {
#ifndef _WIN32
if (desired.is_nop(false /* ignoreAtime */)) {
// Short-circuit completely nop requests as early as possible, without doing
// any additional work to fetch current metadata.
@ -555,8 +556,15 @@ ImmediateFuture<struct stat> FileInode::setattr(
return runWhileMaterialized(
std::move(state), nullptr, setAttrs, fetchContext);
}
#else
(void)desired;
(void)fetchContext;
// neither overlay access nor Inode metadata table is supported on Windows
return makeImmediateFutureWith([]() -> struct stat { NOT_IMPLEMENTED(); });
#endif
}
#ifndef _WIN32
ImmediateFuture<std::string> FileInode::readlink(
const ObjectFetchContextPtr& fetchContext,
CacheHint cacheHint) {
@ -891,6 +899,9 @@ ImmediateFuture<folly::Unit> FileInode::fallocate(
ImmediateFuture<string> FileInode::readAll(
const ObjectFetchContextPtr& fetchContext,
CacheHint cacheHint) {
// TODO: calling this on Windows with a non ProjFS filesystem is likely to
// deadlock Eden. diff calls into this. So `hg status` on non ProjFS mounts
// is likely to hang things.
auto interest = BlobCache::Interest::LikelyNeededAgain;
switch (cacheHint) {
case CacheHint::NotNeededAgain:
@ -942,6 +953,96 @@ ImmediateFuture<string> FileInode::readAll(
});
}
ImmediateFuture<std::tuple<BufVec, bool>>
FileInode::read(size_t size, off_t off, const ObjectFetchContextPtr& context) {
#ifndef _WIN32
XDCHECK_GE(off, 0);
return runWhileDataLoaded(
LockedState{this},
BlobCache::Interest::WantHandle,
// This function is only called by FUSE.
context,
nullptr,
[size, off, self = inodePtrFromThis()](
LockedState&& state,
std::shared_ptr<const Blob> blob) -> std::tuple<BufVec, bool> {
SCOPE_SUCCESS {
self->updateAtimeLocked(*state);
};
// Materialized either before or during blob load.
if (state->tag == State::MATERIALIZED_IN_OVERLAY) {
// TODO(xavierd): For materialized files, only return EOF when
// read returned no bytes. This will force some FS Channel
// (like NFS) to issue at least 2 read calls: one for reading
// the entire file, and the second one to get the EOF bit.
auto buf = self->getOverlayFileAccess(state)->read(*self, size, off);
auto eof = size != 0 && buf->empty();
return {std::move(buf), eof};
}
// runWhileDataLoaded() ensures that the state is either
// MATERIALIZED_IN_OVERLAY or BLOB_NOT_LOADING
XDCHECK_EQ(state->tag, State::BLOB_NOT_LOADING);
XDCHECK(blob) << "blob missing after load completed";
state->readByteRanges.add(off, off + size);
if (state->readByteRanges.covers(0, blob->getSize())) {
XLOG(DBG4) << "Inode " << self->getNodeId()
<< " dropping interest for blob " << blob->getHash()
<< " because it's been fully read.";
state->interestHandle.reset();
state->readByteRanges.clear();
}
auto buf = blob->getContents();
folly::io::Cursor cursor(&buf);
if (!cursor.canAdvance(off)) {
// Seek beyond EOF. Return an empty result.
return {BufVec{folly::IOBuf::wrapBuffer("", 0)}, true};
}
cursor.skip(off);
std::unique_ptr<folly::IOBuf> result;
cursor.cloneAtMost(result, size);
return {BufVec{std::move(result)}, cursor.isAtEnd()};
});
#else
(void)size;
(void)off;
(void)context;
// TODO: overlay access not available on Windows.
return makeImmediateFutureWith(
[]() -> std::tuple<BufVec, bool> { NOT_IMPLEMENTED(); });
#endif
}
ImmediateFuture<size_t> FileInode::write(
BufVec&& buf,
off_t off,
const ObjectFetchContextPtr& fetchContext) {
#ifndef _WIN32
return runWhileMaterialized(
LockedState{this},
nullptr,
[buf = std::move(buf), off, self = inodePtrFromThis()](
LockedState&& state) {
auto vec = buf->getIov();
return self->writeImpl(state, vec.data(), vec.size(), off);
},
fetchContext);
#else
(void)buf;
(void)off;
(void)fetchContext;
// TODO: enable writing on Windows, overlay access is not available.
return makeImmediateFutureWith([]() -> size_t { NOT_IMPLEMENTED(); });
#endif
}
#ifdef _WIN32
void FileInode::materialize() {
{
@ -1007,64 +1108,6 @@ ImmediateFuture<folly::Unit> FileInode::ensureMaterialized(
fetchContext);
}
ImmediateFuture<std::tuple<BufVec, bool>>
FileInode::read(size_t size, off_t off, const ObjectFetchContextPtr& context) {
XDCHECK_GE(off, 0);
return runWhileDataLoaded(
LockedState{this},
BlobCache::Interest::WantHandle,
// This function is only called by FUSE.
context,
nullptr,
[size, off, self = inodePtrFromThis()](
LockedState&& state,
std::shared_ptr<const Blob> blob) -> std::tuple<BufVec, bool> {
SCOPE_SUCCESS {
self->updateAtimeLocked(*state);
};
// Materialized either before or during blob load.
if (state->tag == State::MATERIALIZED_IN_OVERLAY) {
// TODO(xavierd): For materialized files, only return EOF when
// read returned no bytes. This will force some FS Channel
// (like NFS) to issue at least 2 read calls: one for reading
// the entire file, and the second one to get the EOF bit.
auto buf = self->getOverlayFileAccess(state)->read(*self, size, off);
auto eof = size != 0 && buf->empty();
return {std::move(buf), eof};
}
// runWhileDataLoaded() ensures that the state is either
// MATERIALIZED_IN_OVERLAY or BLOB_NOT_LOADING
XDCHECK_EQ(state->tag, State::BLOB_NOT_LOADING);
XDCHECK(blob) << "blob missing after load completed";
state->readByteRanges.add(off, off + size);
if (state->readByteRanges.covers(0, blob->getSize())) {
XLOG(DBG4) << "Inode " << self->getNodeId()
<< " dropping interest for blob " << blob->getHash()
<< " because it's been fully read.";
state->interestHandle.reset();
state->readByteRanges.clear();
}
auto buf = blob->getContents();
folly::io::Cursor cursor(&buf);
if (!cursor.canAdvance(off)) {
// Seek beyond EOF. Return an empty result.
return {BufVec{folly::IOBuf::wrapBuffer("", 0)}, true};
}
cursor.skip(off);
std::unique_ptr<folly::IOBuf> result;
cursor.cloneAtMost(result, size);
return {BufVec{std::move(result)}, cursor.isAtEnd()};
});
}
size_t FileInode::writeImpl(
LockedState& state,
const struct iovec* iov,
@ -1083,21 +1126,6 @@ size_t FileInode::writeImpl(
return xfer;
}
ImmediateFuture<size_t> FileInode::write(
BufVec&& buf,
off_t off,
const ObjectFetchContextPtr& fetchContext) {
return runWhileMaterialized(
LockedState{this},
nullptr,
[buf = std::move(buf), off, self = inodePtrFromThis()](
LockedState&& state) {
auto vec = buf->getIov();
return self->writeImpl(state, vec.data(), vec.size(), off);
},
fetchContext);
}
ImmediateFuture<size_t> FileInode::write(
folly::StringPiece data,
off_t off,

16
eden/fs/inodes/FileInode.h
View File

@ -159,11 +159,11 @@ class FileInode final : public InodeBaseMetadata<FileInodeState> {
mode_t initialMode,
const InodeTimestamps& initialTimestamps);
#ifndef _WIN32
ImmediateFuture<struct stat> setattr(
const DesiredMetadata& desired,
const ObjectFetchContextPtr& fetchContext) override;
#ifndef _WIN32
/// Throws InodeError EINVAL if inode is not a symbolic node.
ImmediateFuture<std::string> readlink(
const ObjectFetchContextPtr& fetchContext,
@ -248,12 +248,6 @@ class FileInode final : public InodeBaseMetadata<FileInodeState> {
const ObjectFetchContextPtr& fetchContext,
CacheHint cacheHint = CacheHint::LikelyNeededAgain);
#ifdef _WIN32
// This function will update the FileInode's state as materialized. This is a
// Windows only function. On POSIX systems the write() functions mark a file
// as Materialized.
void materialize();
#else
/**
* Read up to size bytes from the file at the specified offset.
*
@ -272,6 +266,14 @@ class FileInode final : public InodeBaseMetadata<FileInodeState> {
ImmediateFuture<size_t>
write(BufVec&& buf, off_t off, const ObjectFetchContextPtr& fetchContext);
#ifdef _WIN32
// This function will update the FileInode's state as materialized. This is a
// Windows only function. On POSIX systems the write() functions mark a file
// as Materialized.
void materialize();
#else
ImmediateFuture<size_t> write(
folly::StringPiece data,
off_t off,

2
eden/fs/inodes/InodeBase.h
View File

@ -145,12 +145,12 @@ class InodeBase {
virtual ImmediateFuture<struct stat> stat(
const ObjectFetchContextPtr& context) = 0;
#ifndef _WIN32
// See Dispatcher::setattr
virtual ImmediateFuture<struct stat> setattr(
const DesiredMetadata& desired,
const ObjectFetchContextPtr& fetchContext) = 0;
#ifndef _WIN32
FOLLY_NODISCARD folly::Future<folly::Unit>
setxattr(folly::StringPiece name, folly::StringPiece value, int flags);
FOLLY_NODISCARD folly::Future<folly::Unit> removexattr(

66
eden/fs/inodes/NfsDispatcherImpl.cpp
View File

@ -5,8 +5,6 @@
* GNU General Public License version 2.
*/
#ifndef _WIN32
#include "eden/fs/inodes/NfsDispatcherImpl.h"
#include <folly/futures/Future.h>
@ -17,10 +15,27 @@
#include "eden/fs/inodes/TreeInode.h"
#include "eden/fs/nfs/NfsUtils.h"
#include "eden/fs/telemetry/EdenStats.h"
#include "eden/fs/utils/NotImplemented.h"
#include "eden/fs/utils/String.h"
namespace facebook::eden {
ImmediateFuture<struct stat> statHelper(
const InodePtr& inode,
const ObjectFetchContextPtr& context) {
// TODO: stat is not safe to call on windows because it's gonna try to stat
// the working copy. On NFS thats going to cause infinite recursion, and if I
// had to bet probably blue screens. Needs to be fixed before we can call
// stat.
#ifndef _WIN32
return inode->stat(context);
#else
(void)inode;
(void)context;
return makeImmediateFutureWith([]() -> struct stat { NOT_IMPLEMENTED(); });
#endif
}
NfsDispatcherImpl::NfsDispatcherImpl(EdenMount* mount)
: NfsDispatcher(mount->getStats().copy(), mount->getClock()),
mount_(mount),
@ -31,7 +46,7 @@ ImmediateFuture<struct stat> NfsDispatcherImpl::getattr(
const ObjectFetchContextPtr& context) {
return inodeMap_->lookupInode(ino).thenValue(
[context = context.copy()](const InodePtr& inode) {
return inode->stat(context);
return statHelper(inode, context);
});
}
@ -69,7 +84,7 @@ ImmediateFuture<std::tuple<InodeNumber, struct stat>> NfsDispatcherImpl::lookup(
return inode->getOrLoadChild(name, context);
})
.thenValue([context = context.copy()](InodePtr&& inode) {
auto statFut = inode->stat(context);
auto statFut = statHelper(inode, context);
return std::move(statFut).thenValue(
[inode = std::move(inode)](
struct stat stat) -> std::tuple<InodeNumber, struct stat> {
@ -84,7 +99,16 @@ ImmediateFuture<std::string> NfsDispatcherImpl::readlink(
const ObjectFetchContextPtr& context) {
return inodeMap_->lookupFileInode(ino).thenValue(
[context = context.copy()](const FileInodePtr& inode) {
#ifndef _WIN32
return inode->readlink(context);
#else
// todo: enable readlink on Windows
// this would read out of the working copy on windows. not what we
// want on NFS.
(void)inode;
return makeImmediateFutureWith(
[]() -> std::string { NOT_IMPLEMENTED(); });
#endif
});
}
@ -143,7 +167,7 @@ ImmediateFuture<NfsDispatcher::CreateRes> NfsDispatcherImpl::create(
// directory.
// Set dev to 0 as this is unused for a regular file.
auto newFile = inode->mknod(name, mode, 0, InvalidationRequired::No);
auto statFut = newFile->stat(context);
auto statFut = statHelper(newFile, context);
return std::move(statFut).thenValue(
[newFile = std::move(newFile)](struct stat&& stat) {
newFile->incFsRefcount();
@ -167,7 +191,7 @@ ImmediateFuture<NfsDispatcher::MkdirRes> NfsDispatcherImpl::mkdir(
// TODO(xavierd): Modify mkdir to obtain the pre and post stat of the
// directory.
auto newDir = inode->mkdir(name, mode, InvalidationRequired::No);
auto statFut = newDir->stat(context);
auto statFut = statHelper(newDir, context);
return std::move(statFut).thenValue([newDir = std::move(newDir)](
struct stat&& stat) {
newDir->incFsRefcount();
@ -189,7 +213,7 @@ ImmediateFuture<NfsDispatcher::SymlinkRes> NfsDispatcherImpl::symlink(
// TODO(xavierd): Modify symlink to obtain the pre and post stat of the
// directory.
auto symlink = inode->symlink(name, data, InvalidationRequired::No);
auto statFut = symlink->stat(context);
auto statFut = statHelper(symlink, context);
return std::move(statFut).thenValue(
[symlink = std::move(symlink)](struct stat&& stat) {
symlink->incFsRefcount();
@ -214,7 +238,7 @@ ImmediateFuture<NfsDispatcher::MknodRes> NfsDispatcherImpl::mknod(
// TODO(xavierd): Modify mknod to obtain the pre and post stat of the
// directory.
auto newFile = inode->mknod(name, mode, rdev, InvalidationRequired::No);
auto statFut = newFile->stat(context);
auto statFut = statHelper(newFile, context);
return std::move(statFut).thenValue(
[newFile = std::move(newFile)](struct stat&& stat) {
newFile->incFsRefcount();
@ -308,6 +332,7 @@ ImmediateFuture<NfsDispatcher::ReaddirRes> NfsDispatcherImpl::readdirplus(
off_t offset,
uint32_t count,
const ObjectFetchContextPtr& context) {
#ifndef _WIN32
return inodeMap_->lookupTreeInode(dir).thenValue(
[context = context.copy(), offset, count, this](
const TreeInodePtr& inode) {
@ -328,7 +353,7 @@ ImmediateFuture<NfsDispatcher::ReaddirRes> NfsDispatcherImpl::readdirplus(
inode->getOrLoadChild(PathComponent{entry.name}, context)
.thenValue(
[entry, context = context.copy()](InodePtr&& inodep) {
return inodep->stat(context);
return statHelper(inodep, context);
})
.thenTry([&entry](folly::Try<struct stat> st) {
entry.name_attributes = statToPostOpAttr(st);
@ -343,16 +368,35 @@ ImmediateFuture<NfsDispatcher::ReaddirRes> NfsDispatcherImpl::readdirplus(
return ReaddirRes{std::move(dirList), isEof};
});
});
#else
// TODO: implement readdirplus on Windows.
// shouldn't be too hard, but left out for now since we don't use readdir plus
// in production.
(void)dir;
(void)offset;
(void)count;
(void)context;
return makeImmediateFutureWith(
[]() -> NfsDispatcher::ReaddirRes { NOT_IMPLEMENTED(); });
#endif
}
#ifdef _WIN32
// TODO: find a statfs definition for Windows?
struct statfs {};
#endif
ImmediateFuture<struct statfs> NfsDispatcherImpl::statfs(
InodeNumber /*dir*/,
const ObjectFetchContextPtr& /*context*/) {
#ifndef _WIN32
// See the comment in FuseDispatcherImpl::statfs for why we gather the statFs
// from the overlay.
return mount_->getOverlay()->statFs();
#else
// TODO: implement statfs on windows
return makeImmediateFutureWith([]() -> struct statfs { NOT_IMPLEMENTED(); });
#endif
}
} // namespace facebook::eden
#endif

4
eden/fs/inodes/NfsDispatcherImpl.h
View File

@ -7,8 +7,6 @@
#pragma once
#ifndef _WIN32
#include "eden/fs/nfs/NfsDispatcher.h"
namespace facebook::eden {
@ -117,5 +115,3 @@ class NfsDispatcherImpl : public NfsDispatcher {
InodeMap* const inodeMap_;
};
} // namespace facebook::eden
#endif

35
eden/fs/inodes/TreeInode.cpp
View File

@ -36,6 +36,7 @@
#include "eden/fs/model/Tree.h"
#include "eden/fs/model/TreeEntry.h"
#include "eden/fs/model/git/GitIgnoreStack.h"
#include "eden/fs/nfs/NfsDirList.h"
#include "eden/fs/nfs/NfsdRpc.h"
#include "eden/fs/prjfs/Enumerator.h"
#include "eden/fs/service/ThriftUtil.h"
@ -50,6 +51,7 @@
#include "eden/fs/utils/Clock.h"
#include "eden/fs/utils/FaultInjector.h"
#include "eden/fs/utils/ImmediateFuture.h"
#include "eden/fs/utils/NotImplemented.h"
#include "eden/fs/utils/PathFuncs.h"
#include "eden/fs/utils/SystemError.h"
#include "eden/fs/utils/TimeUtil.h"
@ -1198,13 +1200,13 @@ FileInodePtr TreeInode::createImpl(
return inode;
}
#ifndef _WIN32
// Eden doesn't support symlinks on Windows
FileInodePtr TreeInode::symlink(
PathComponentPiece name,
folly::StringPiece symlinkTarget,
InvalidationRequired invalidate) {
#ifndef _WIN32
// Eden doesn't support symlinks on Windows
// symlink creates a newly materialized file in createImpl. We count this as
// an inode materialization event to publish to TraceBus, which we begin
// timing here before the parent tree inode materializes
@ -1225,8 +1227,13 @@ FileInodePtr TreeInode::symlink(
invalidate,
startTime);
}
}
#else
(void)name;
(void)symlinkTarget;
(void)invalidate;
NOT_IMPLEMENTED();
#endif
}
FileInodePtr TreeInode::mknod(
PathComponentPiece name,
@ -2220,7 +2227,6 @@ void TreeInode::TreeRenameLocks::lockDestChild(PathComponentPiece destName) {
}
}
#ifndef _WIN32
template <typename Fn>
bool TreeInode::readdirImpl(
off_t off,
@ -2272,8 +2278,12 @@ bool TreeInode::readdirImpl(
// serially stat() every entry. Since stat() returns a file's size and a
// directory's entry count in the st_nlink field, treat readdir() as a signal
// that we may want to prefetch metadata for all children.
#ifndef _WIN32
// TODO: enable readdir prefetching on Windows
considerReaddirPrefetch(context);
#else
(void)context;
#endif
// Possible offset values are:
// 0: start at the beginning
// 1: start after .
@ -2329,6 +2339,7 @@ bool TreeInode::readdirImpl(
return true;
}
#ifndef _WIN32
FuseDirList TreeInode::fuseReaddir(
FuseDirList&& list,
off_t off,
@ -2344,6 +2355,8 @@ FuseDirList TreeInode::fuseReaddir(
return std::move(list);
}
#endif // _WIN32
std::tuple<NfsDirList, bool> TreeInode::nfsReaddir(
NfsDirList&& list,
off_t off,
@ -2358,7 +2371,6 @@ std::tuple<NfsDirList, bool> TreeInode::nfsReaddir(
return {std::move(list), isEof};
}
#endif // _WIN32
InodeMap* TreeInode::getInodeMap() const {
return getMount()->getInodeMap();
@ -4599,11 +4611,10 @@ void TreeInode::doPrefetch(
});
}
#ifndef _WIN32
ImmediateFuture<struct stat> TreeInode::setattr(
const DesiredMetadata& desired,
const ObjectFetchContextPtr& /*fetchContext*/) {
#ifndef _WIN32
struct stat result(getMount()->initStatData());
result.st_ino = getNodeId().get();
@ -4639,8 +4650,14 @@ ImmediateFuture<struct stat> TreeInode::setattr(
// Update Journal
updateJournal();
return result;
#else
(void)desired;
// Inode metatdata table is not on Windows
return makeImmediateFutureWith([]() -> struct stat { NOT_IMPLEMENTED(); });
#endif
}
#ifndef _WIN32
ImmediateFuture<std::vector<std::string>> TreeInode::listxattr() {
return std::vector<std::string>{};
}

5
eden/fs/inodes/TreeInode.h
View File

@ -115,11 +115,11 @@ class TreeInode final : public InodeBaseMetadata<DirContents> {
ImmediateFuture<struct stat> stat(
const ObjectFetchContextPtr& context) override;
#ifndef _WIN32
ImmediateFuture<struct stat> setattr(
const DesiredMetadata& desired,
const ObjectFetchContextPtr& fetchContext) override;
#ifndef _WIN32
ImmediateFuture<std::vector<std::string>> listxattr() override;
ImmediateFuture<std::string> getxattr(
folly::StringPiece name,
@ -195,7 +195,7 @@ class TreeInode final : public InodeBaseMetadata<DirContents> {
FuseDirList&& list,
off_t off,
const ObjectFetchContextPtr& context);
#endif
/**
* Populate the list with as many directory entries as possible starting from
* the inode start.
@ -207,7 +207,6 @@ class TreeInode final : public InodeBaseMetadata<DirContents> {
NfsDirList&& list,
off_t off,
const ObjectFetchContextPtr& context);
#endif
const folly::Synchronized<TreeInodeState>& getContents() const {
return contents_;

4
eden/fs/nfs/NfsDirList.cpp
View File

@ -5,8 +5,6 @@
* GNU General Public License version 2.
*/
#ifndef _WIN32
#include "eden/fs/nfs/NfsDirList.h"
#include <variant>
#include "eden/fs/nfs/NfsdRpc.h"
@ -85,5 +83,3 @@ bool NfsDirList::add(
}
} // namespace facebook::eden
#endif