diff --git a/eden/fs/inodes/FileData.cpp b/eden/fs/inodes/FileData.cpp deleted file mode 100644 index 2d1c1960a4..0000000000 --- a/eden/fs/inodes/FileData.cpp +++ /dev/null @@ -1,479 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ -#include "FileData.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "eden/fs/fuse/BufVec.h" -#include "eden/fs/fuse/MountPoint.h" -#include "eden/fs/fuse/fuse_headers.h" -#include "eden/fs/inodes/EdenMount.h" -#include "eden/fs/inodes/FileInode.h" -#include "eden/fs/inodes/Overlay.h" -#include "eden/fs/model/Blob.h" -#include "eden/fs/model/Hash.h" -#include "eden/fs/store/ObjectStore.h" -#include "eden/fs/utils/Bug.h" -#include "eden/fs/utils/XAttr.h" - -using folly::ByteRange; -using folly::checkUnixError; -using folly::Future; -using folly::makeFuture; -using folly::Unit; -using folly::StringPiece; - -namespace facebook { -namespace eden { - -FileData::FileData(FileInode* inode) : inode_(inode) {} - -// Conditionally updates target with either the value provided by -// the caller, or with the current time value, depending on the value -// of the flags in to_set. Valid flag values are defined in fuse_lowlevel.h -// and have symbolic names matching FUSE_SET_*. -// useAttrFlag is the bitmask that indicates whether we should use the value -// from wantedTimeSpec. useNowFlag is the bitmask that indicates whether we -// should use the current time instead. -// If neither flag is present, we will preserve the current value in target. -static void resolveTimeForSetAttr( - struct timespec& target, - int to_set, - int useAttrFlag, - int useNowFlag, - const struct timespec& wantedTimeSpec) { - if (to_set & useAttrFlag) { - target = wantedTimeSpec; - } else if (to_set & useNowFlag) { - clock_gettime(CLOCK_REALTIME, &target); - } -} - -// Valid values for to_set are found in fuse_lowlevel.h and have symbolic -// names matching FUSE_SET_*. -struct stat FileData::setAttr(const struct stat& attr, int to_set) { - auto state = inode_->state_.wlock(); - - CHECK(state->file) << "MUST have a materialized file at this point"; - - // We most likely need the current information to apply the requested - // changes below, so just fetch it here first. - struct stat currentStat; - checkUnixError(fstat(state->file.fd(), ¤tStat)); - - if (to_set & FUSE_SET_ATTR_SIZE) { - checkUnixError( - ftruncate(state->file.fd(), attr.st_size + Overlay::kHeaderLength)); - } - - if (to_set & (FUSE_SET_ATTR_UID | FUSE_SET_ATTR_GID)) { - if ((to_set & FUSE_SET_ATTR_UID && attr.st_uid != currentStat.st_uid) || - (to_set & FUSE_SET_ATTR_GID && attr.st_gid != currentStat.st_gid)) { - folly::throwSystemErrorExplicit( - EACCES, "changing the owner/group is not supported"); - } - - // Otherwise: there is no change - } - - if (to_set & FUSE_SET_ATTR_MODE) { - // The mode data is stored only in inode_->state_. - // (We don't set mode bits on the overlay file as that may incorrectly - // prevent us from reading or writing the overlay data). - // Make sure we preserve the file type bits, and only update permissions. - state->mode = (state->mode & S_IFMT) | (07777 & attr.st_mode); - } - - if (to_set & (FUSE_SET_ATTR_ATIME | FUSE_SET_ATTR_MTIME | - FUSE_SET_ATTR_ATIME_NOW | FUSE_SET_ATTR_MTIME_NOW)) { - // Changing various time components. - // Element 0 is the atime, element 1 is the mtime. - struct timespec times[2] = {currentStat.st_atim, currentStat.st_mtim}; - - resolveTimeForSetAttr( - times[0], - to_set, - FUSE_SET_ATTR_ATIME, - FUSE_SET_ATTR_ATIME_NOW, - attr.st_atim); - - resolveTimeForSetAttr( - times[1], - to_set, - FUSE_SET_ATTR_MTIME, - FUSE_SET_ATTR_MTIME_NOW, - attr.st_mtim); - - checkUnixError(futimens(state->file.fd(), times)); - } - - // We need to return the now-current stat information for this file. - struct stat returnedStat; - checkUnixError(fstat(state->file.fd(), &returnedStat)); - returnedStat.st_mode = state->mode; - returnedStat.st_size -= Overlay::kHeaderLength; - - return returnedStat; -} - -struct stat FileData::stat() { - auto st = inode_->getMount()->getMountPoint()->initStatData(); - st.st_nlink = 1; - - auto state = inode_->state_.rlock(); - - if (state->file) { - // stat() the overlay file. - // - // TODO: We need to get timestamps accurately here. - // The timestamps on the underlying file are not correct, because we keep - // file_ open for a long time, and do not close it when FUSE file handles - // close. (Timestamps are typically only updated on close operations.) - // This results our reported timestamps not changing correctly after the - // file is changed through FUSE APIs. - // - // We probably should update the overlay file to include a header, - // so we can store the atime, mtime, and ctime in the header data. - // Otherwise we won't be able to report the ctime accurately if we just - // keep using the overlay file timestamps. - checkUnixError(fstat(state->file.fd(), &st)); - - if (st.st_size < Overlay::kHeaderLength) { - auto filePath = inode_->getLocalPath(); - EDEN_BUG() << "Overlay file " << inode_->getLocalPath() - << " is too short for header: size=" << st.st_size; - } - - st.st_size -= Overlay::kHeaderLength; - st.st_mode = state->mode; - st.st_rdev = state->rdev; - - return st; - } - - CHECK(state->blob); - st.st_mode = state->mode; - - auto buf = state->blob->getContents(); - st.st_size = buf.computeChainDataLength(); - - // Report atime, mtime, and ctime as the time when we first loaded this - // FileInode. It hasn't been materialized yet, so this is a reasonble time - // to use. Once it is materialized we use the timestamps on the underlying - // overlay file, which the kernel keeps up-to-date. - auto epochTime = state->creationTime.time_since_epoch(); - auto epochSeconds = - std::chrono::duration_cast(epochTime); - st.st_atime = epochSeconds.count(); - st.st_mtime = epochSeconds.count(); - st.st_ctime = epochSeconds.count(); -#if defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || \ - _POSIX_C_SOURCE >= 200809L || _XOPEN_SOURCE >= 700 - auto nsec = std::chrono::duration_cast( - epochTime - epochSeconds); - st.st_atim.tv_nsec = nsec.count(); - st.st_mtim.tv_nsec = nsec.count(); - st.st_ctim.tv_nsec = nsec.count(); -#endif - - // NOTE: we don't set rdev to anything special here because we - // don't support committing special device nodes. - - return st; -} - -void FileData::flush(uint64_t /* lock_owner */) { - // We have no write buffers, so there is nothing for us to flush, - // but let's take this opportunity to update the sha1 attribute. - auto state = inode_->state_.wlock(); - if (state->file && !state->sha1Valid) { - recomputeAndStoreSha1(state); - } -} - -void FileData::fsync(bool datasync) { - auto state = inode_->state_.wlock(); - if (!state->file) { - // If we don't have an overlay file then we have nothing to sync. - return; - } - - auto res = -#ifndef __APPLE__ - datasync ? ::fdatasync(state->file.fd()) : -#endif - ::fsync(state->file.fd()); - checkUnixError(res); - - // let's take this opportunity to update the sha1 attribute. - if (!state->sha1Valid) { - recomputeAndStoreSha1(state); - } -} - -std::unique_ptr FileData::readIntoBuffer(size_t size, off_t off) { - auto state = inode_->state_.rlock(); - - if (state->file) { - auto buf = folly::IOBuf::createCombined(size); - auto res = ::pread( - state->file.fd(), - buf->writableBuffer(), - size, - off + Overlay::kHeaderLength); - - checkUnixError(res); - buf->append(res); - return buf; - } - - auto buf = state->blob->getContents(); - folly::io::Cursor cursor(&buf); - - if (!cursor.canAdvance(off)) { - // Seek beyond EOF. Return an empty result. - return folly::IOBuf::wrapBuffer("", 0); - } - - cursor.skip(off); - - std::unique_ptr result; - cursor.cloneAtMost(result, size); - return result; -} - -std::string FileData::readAll() { - auto state = inode_->state_.rlock(); - if (state->file) { - std::string result; - auto rc = lseek(state->file.fd(), Overlay::kHeaderLength, SEEK_SET); - folly::checkUnixError(rc, "unable to seek in materialized FileData"); - folly::readFile(state->file.fd(), result); - return result; - } - - const auto& contentsBuf = state->blob->getContents(); - folly::io::Cursor cursor(&contentsBuf); - return cursor.readFixedString(contentsBuf.computeChainDataLength()); -} - -fusell::BufVec FileData::read(size_t size, off_t off) { - auto buf = readIntoBuffer(size, off); - return fusell::BufVec(std::move(buf)); -} - -size_t FileData::write(fusell::BufVec&& buf, off_t off) { - auto state = inode_->state_.wlock(); - if (!state->file) { - // Not open for write - folly::throwSystemErrorExplicit(EINVAL); - } - - state->sha1Valid = false; - auto vec = buf.getIov(); - auto xfer = ::pwritev( - state->file.fd(), vec.data(), vec.size(), off + Overlay::kHeaderLength); - checkUnixError(xfer); - return xfer; -} - -size_t FileData::write(folly::StringPiece data, off_t off) { - auto state = inode_->state_.wlock(); - if (!state->file) { - // Not open for write - folly::throwSystemErrorExplicit(EINVAL); - } - - state->sha1Valid = false; - auto xfer = ::pwrite( - state->file.fd(), data.data(), data.size(), off + Overlay::kHeaderLength); - checkUnixError(xfer); - return xfer; -} - -Future FileData::ensureDataLoaded() { - auto state = inode_->state_.wlock(); - - if (!state->hash.hasValue()) { - // We should always have the file open if we are materialized. - CHECK(state->file); - return makeFuture(); - } - - if (state->blob) { - DCHECK_EQ(state->blob->getHash(), state->hash.value()); - return makeFuture(); - } - - // Load the blob data. - auto blobFuture = getObjectStore()->getBlob(state->hash.value()); - - // TODO: We really should defer this using a Future rather than calling get() - // here and blocking until the load completes. However, for that to work we - // will need to add some extra data tracking whether or not we are already in - // the process of loading the data. We need to avoid multiple threads all - // trying to load the data at the same time. - // - // For now doing a blocking load with the inode_->state_ lock held ensures - // that only one thread can load the data at a time. It's pretty unfortunate - // to block with the lock held, though :-( - state->blob = blobFuture.get(); - return makeFuture(); -} - -Future FileData::materializeForWrite(int openFlags) { - auto state = inode_->state_.wlock(); - - // If we already have a materialized overlay file then we don't - // need to do much - if (state->file) { - CHECK(!state->hash.hasValue()); - if ((openFlags & O_TRUNC) != 0) { - // truncating a file that we already have open - state->sha1Valid = false; - checkUnixError(ftruncate(state->file.fd(), Overlay::kHeaderLength)); - auto emptySha1 = Hash::sha1(ByteRange{}); - storeSha1(state, emptySha1); - } else { - // no truncate option,overlay file contain old header - // we have to update only header but not contents - } - return makeFuture(); - } - - // Add header to the overlay File. - struct timespec zeroTime = {0, 0}; - auto header = Overlay::createHeader( - Overlay::kHeaderIdentifierFile, - Overlay::kHeaderVersion, - zeroTime, - zeroTime, - zeroTime); - auto iov = header.getIov(); - - // We must not be materialized yet - CHECK(state->hash.hasValue()); - - Hash sha1; - auto filePath = inode_->getLocalPath(); - - if ((openFlags & O_TRUNC) != 0) { - folly::writeFileAtomic(filePath.stringPiece(), iov.data(), iov.size()); - state->file = Overlay::openFile(filePath.stringPiece()); - sha1 = Hash::sha1(ByteRange{}); - } else { - if (!state->blob) { - // TODO: Load the blob using the non-blocking Future APIs. - // However, just as in ensureDataLoaded() above we will also need - // to add a mechanism to wait for already in-progress loads. - auto blobFuture = getObjectStore()->getBlob(state->hash.value()); - state->blob = blobFuture.get(); - } - - // Write the blob contents out to the overlay - auto contents = state->blob->getContents().getIov(); - iov.insert(iov.end(), contents.begin(), contents.end()); - - folly::writeFileAtomic( - filePath.stringPiece(), iov.data(), iov.size(), 0600); - state->file = Overlay::openFile(filePath.stringPiece()); - - sha1 = getObjectStore()->getSha1ForBlob(state->hash.value()); - } - - // Copy and apply the sha1 to the new file. This saves us from - // recomputing it again in the case that something opens the file - // read/write and closes it without changing it. - storeSha1(state, sha1); - - // Update the FileInode to indicate that we are materialized now - state->blob.reset(); - state->hash = folly::none; - - return makeFuture(); -} - -Hash FileData::getSha1() { - auto state = inode_->state_.wlock(); - if (state->file) { - std::string shastr; - if (state->sha1Valid) { - shastr = fgetxattr(state->file.fd(), kXattrSha1); - } - if (shastr.empty()) { - return recomputeAndStoreSha1(state); - } else { - return Hash(shastr); - } - } - - CHECK(state->hash.hasValue()); - return getObjectStore()->getSha1ForBlob(state->hash.value()); -} - -ObjectStore* FileData::getObjectStore() const { - return inode_->getMount()->getObjectStore(); -} - -Hash FileData::recomputeAndStoreSha1( - const folly::Synchronized::LockedPtr& state) { - uint8_t buf[8192]; - off_t off = Overlay::kHeaderLength; - SHA_CTX ctx; - SHA1_Init(&ctx); - - while (true) { - // Using pread here so that we don't move the file position; - // the file descriptor is shared between multiple file handles - // and while we serialize the requests to FileData, it seems - // like a good property of this function to avoid changing that - // state. - auto len = folly::preadNoInt(state->file.fd(), buf, sizeof(buf), off); - if (len == 0) { - break; - } - if (len == -1) { - folly::throwSystemError(); - } - SHA1_Update(&ctx, buf, len); - off += len; - } - - uint8_t digest[SHA_DIGEST_LENGTH]; - SHA1_Final(digest, &ctx); - auto sha1 = Hash(folly::ByteRange(digest, sizeof(digest))); - storeSha1(state, sha1); - return sha1; -} - -void FileData::storeSha1( - const folly::Synchronized::LockedPtr& state, - Hash sha1) { - try { - fsetxattr(state->file.fd(), kXattrSha1, sha1.toString()); - state->sha1Valid = true; - } catch (const std::exception& ex) { - // If something goes wrong storing the attribute just log a warning - // and leave sha1Valid as false. We'll have to recompute the value - // next time we need it. - XLOG(WARNING) << "error setting SHA1 attribute in the overlay: " - << folly::exceptionStr(ex); - } -} -} -} diff --git a/eden/fs/inodes/FileData.h b/eden/fs/inodes/FileData.h deleted file mode 100644 index b40756dd38..0000000000 --- a/eden/fs/inodes/FileData.h +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright (c) 2016-present, Facebook, Inc. - * All rights reserved. - * - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. An additional grant - * of patent rights can be found in the PATENTS file in the same directory. - * - */ -#pragma once -#include -#include -#include -#include -#include -#include "eden/fs/inodes/FileInode.h" -#include "eden/fs/model/Tree.h" - -namespace folly { -template -class Optional; -} - -namespace facebook { -namespace eden { -namespace fusell { -class BufVec; -} - -class Blob; -class Hash; -class ObjectStore; -class Overlay; - -/** - * FileData stores information about a file contents. - * - * The data may be lazily loaded from the EdenMount's ObjectStore only when it - * is needed. - * - * FileData objects are tracked via shared_ptr. FileInode and FileHandle - * objects maintain references to them. FileData objects never outlive - * the FileInode to which they belong. - */ -class FileData { - public: - explicit FileData(FileInode* inode); - - /** - * Read up to size bytes from the file at the specified offset. - * - * Returns an IOBuf containing the data. This may return fewer bytes than - * requested. If the specified offset is at or past the end of the buffer an - * empty IOBuf will be returned. Otherwise between 1 and size bytes will be - * returned. If fewer than size bytes are returned this does *not* guarantee - * that the end of the file was reached. - * - * May throw exceptions on error. - */ - std::unique_ptr readIntoBuffer(size_t size, off_t off); - fusell::BufVec read(size_t size, off_t off); - size_t write(fusell::BufVec&& buf, off_t off); - size_t write(folly::StringPiece data, off_t off); - struct stat stat(); - void flush(uint64_t lock_owner); - void fsync(bool datasync); - - /// Change attributes for this inode. - // attr is a standard struct stat. Only the members indicated - // by to_set are valid. Defined values for the to_set flags - // are found in the fuse_lowlevel.h header file and have symbolic - // names matching FUSE_SET_*. - struct stat setAttr(const struct stat& attr, int to_set); - - /// Returns the sha1 hash of the content. - Hash getSha1(); - - /** - * Read the entire file contents, and return them as a string. - * - * Note that this API generally should only be used for fairly small files. - */ - std::string readAll(); - - /** - * Materialize the file data. - * openFlags has the same meaning as the flags parameter to - * open(2). Materialization depends on the write mode specified - * in those flags; if we are writing to the file then we need to - * copy it locally to the overlay. If we are truncating we just - * need to create an empty file in the overlay. Otherwise we - * need to go out to the LocalStore to obtain the backing data. - */ - FOLLY_NODISCARD folly::Future materializeForWrite(int openFlags); - - /** - * Load the file data so it can be used for reading. - * - * If this file is materialized, this opens it's file in the overlay. - * If the file is not materialized, this loads the Blob data from the - * ObjectStore. - */ - FOLLY_NODISCARD folly::Future ensureDataLoaded(); - - private: - ObjectStore* getObjectStore() const; - - /// Recompute the SHA1 content hash of the open file_. - Hash recomputeAndStoreSha1( - const folly::Synchronized::LockedPtr& state); - void storeSha1( - const folly::Synchronized::LockedPtr& state, - Hash sha1); - - /** - * The FileInode that this FileData object belongs to. - * - * This pointer never changes once a FileData object is constructed. A - * FileData always belongs to the same FileInode. Therefore it is safe to - * access this pointer without locking. - */ - FileInode* const inode_{nullptr}; - -}; -} -} diff --git a/eden/fs/inodes/FileHandle.cpp b/eden/fs/inodes/FileHandle.cpp index e95fcda50f..d56e442335 100644 --- a/eden/fs/inodes/FileHandle.cpp +++ b/eden/fs/inodes/FileHandle.cpp @@ -11,7 +11,6 @@ #include #include "eden/fs/inodes/EdenMount.h" -#include "eden/fs/inodes/FileData.h" #include "eden/fs/inodes/FileInode.h" #include "eden/fs/inodes/TreeInode.h" #include "eden/fs/store/LocalStore.h" @@ -19,19 +18,8 @@ namespace facebook { namespace eden { -FileHandle::FileHandle( - FileInodePtr inode, - std::shared_ptr data, - int flags) - : inode_(std::move(inode)), data_(std::move(data)), openFlags_(flags) {} - -FileHandle::~FileHandle() { - // Must reset the data point prior to calling fileHandleDidClose, - // otherwise it will see a use count that is too high and won't - // reclaim resources soon enough. - data_.reset(); - inode_->fileHandleDidClose(); -} +FileHandle::FileHandle(FileInodePtr inode, int flags) + : inode_(std::move(inode)), openFlags_(flags) {} folly::Future FileHandle::getattr() { FB_LOGF( @@ -64,7 +52,7 @@ bool FileHandle::isSeekable() const { folly::Future FileHandle::read(size_t size, off_t off) { FB_LOGF( inode_->getMount()->getLogger(), DBG7, "read({})", inode_->getNodeId()); - return data_->read(size, off); + return inode_->read(size, off); } folly::Future FileHandle::write(fusell::BufVec&& buf, off_t off) { @@ -77,7 +65,7 @@ folly::Future FileHandle::write(fusell::BufVec&& buf, off_t off) { }; FB_LOGF( inode_->getMount()->getLogger(), DBG7, "write({})", inode_->getNodeId()); - return data_->write(std::move(buf), off); + return inode_->write(std::move(buf), off); } folly::Future FileHandle::write(folly::StringPiece str, off_t off) { @@ -90,20 +78,20 @@ folly::Future FileHandle::write(folly::StringPiece str, off_t off) { }; FB_LOGF( inode_->getMount()->getLogger(), DBG7, "write({})", inode_->getNodeId()); - return data_->write(str, off); + return inode_->write(str, off); } folly::Future FileHandle::flush(uint64_t lock_owner) { FB_LOGF( inode_->getMount()->getLogger(), DBG7, "flush({})", inode_->getNodeId()); - data_->flush(lock_owner); + inode_->flush(lock_owner); return folly::Unit{}; } folly::Future FileHandle::fsync(bool datasync) { FB_LOGF( inode_->getMount()->getLogger(), DBG7, "fsync({})", inode_->getNodeId()); - data_->fsync(datasync); + inode_->fsync(datasync); return folly::Unit{}; } } diff --git a/eden/fs/inodes/FileHandle.h b/eden/fs/inodes/FileHandle.h index 575eb28fb5..90211562d1 100644 --- a/eden/fs/inodes/FileHandle.h +++ b/eden/fs/inodes/FileHandle.h @@ -23,9 +23,8 @@ class FileHandle : public fusell::FileHandle { public: explicit FileHandle( FileInodePtr inode, - std::shared_ptr data, int flags); - ~FileHandle() override; + ~FileHandle() override {} folly::Future getattr() override; folly::Future setattr( diff --git a/eden/fs/inodes/FileInode.cpp b/eden/fs/inodes/FileInode.cpp index 19e0f28322..e9d47d872c 100644 --- a/eden/fs/inodes/FileInode.cpp +++ b/eden/fs/inodes/FileInode.cpp @@ -9,8 +9,13 @@ */ #include "eden/fs/inodes/FileInode.h" +#include +#include +#include +#include +#include +#include "eden/fs/fuse/MountPoint.h" #include "eden/fs/inodes/EdenMount.h" -#include "eden/fs/inodes/FileData.h" #include "eden/fs/inodes/FileHandle.h" #include "eden/fs/inodes/InodeError.h" #include "eden/fs/inodes/Overlay.h" @@ -19,15 +24,18 @@ #include "eden/fs/model/Hash.h" #include "eden/fs/store/BlobMetadata.h" #include "eden/fs/store/ObjectStore.h" +#include "eden/fs/utils/Bug.h" #include "eden/fs/utils/XAttr.h" using folly::checkUnixError; using folly::Future; using folly::makeFuture; using folly::StringPiece; +using folly::Unit; using std::shared_ptr; using std::string; using std::vector; +using folly::ByteRange; namespace facebook { namespace eden { @@ -36,10 +44,7 @@ FileInode::State::State( FileInode* inode, mode_t m, const folly::Optional& h) - : data(std::make_shared(inode)), - mode(m), - creationTime(std::chrono::system_clock::now()), - hash(h) { + : mode(m), creationTime(std::chrono::system_clock::now()), hash(h) { if (!h.hasValue()) { auto filePath = inode->getLocalPath(); file = Overlay::openFile(filePath.c_str()); @@ -51,8 +56,7 @@ FileInode::State::State( mode_t m, folly::File&& file, dev_t rdev) - : data(std::make_shared(inode)), - mode(m), + : mode(m), rdev(rdev), creationTime(std::chrono::system_clock::now()), file(std::move(file)) {} @@ -82,15 +86,13 @@ FileInode::FileInode( state_(folly::in_place, this, mode, std::move(file), rdev) {} folly::Future FileInode::getattr() { - auto data = getOrLoadData(); - // Future optimization opportunity: right now, if we have not already // materialized the data from the entry, we have to materialize it // from the store. If we augmented our metadata we could avoid this, // and this would speed up operations like `ls`. - return data->ensureDataLoaded().then([ self = inodePtrFromThis(), data ]() { + return ensureDataLoaded().then([self = inodePtrFromThis()]() { auto attr = fusell::Dispatcher::Attr{self->getMount()->getMountPoint()}; - attr.st = data->stat(); + attr.st = self->stat(); attr.st.st_ino = self->getNodeId(); return attr; }); @@ -99,7 +101,6 @@ folly::Future FileInode::getattr() { folly::Future FileInode::setattr( const struct stat& attr, int to_set) { - auto data = getOrLoadData(); int openFlags = O_RDWR; // Minor optimization: if we know that the file is being completed truncated @@ -109,13 +110,13 @@ folly::Future FileInode::setattr( openFlags |= O_TRUNC; } - return data->materializeForWrite(openFlags).then( - [ self = inodePtrFromThis(), data, attr, to_set ]() { + return materializeForWrite(openFlags).then( + [ self = inodePtrFromThis(), attr, to_set ]() { self->materializeInParent(); auto result = fusell::Dispatcher::Attr{self->getMount()->getMountPoint()}; - result.st = data->setAttr(attr, to_set); + result.st = self->setAttr(attr, to_set); result.st.st_ino = self->getNodeId(); auto path = self->getPath(); @@ -129,89 +130,68 @@ folly::Future FileInode::setattr( } folly::Future FileInode::readlink() { - shared_ptr data; { auto state = state_.wlock(); if (!S_ISLNK(state->mode)) { // man 2 readlink says: EINVAL The named file is not a symbolic link. throw InodeError(EINVAL, inodePtrFromThis(), "not a symlink"); } - - data = getOrLoadData(state); } // The symlink contents are simply the file contents! - return data->ensureDataLoaded().then( - [ self = inodePtrFromThis(), data ]() { return data->readAll(); }); -} - -std::shared_ptr FileInode::getOrLoadData() { - return getOrLoadData(state_.wlock()); -} - -std::shared_ptr FileInode::getOrLoadData( - const folly::Synchronized::LockedPtr& state) { - if (!state->data) { - state->data = std::make_shared(this); - } - - return state->data; + return ensureDataLoaded().then([self = inodePtrFromThis()]() { + return self->readAll(); + }); } void FileInode::fileHandleDidClose() { { - auto state = state_.wlock(); - if (state->data.unique()) { - // We're the only remaining user, no need to keep it around - state->data.reset(); - } + // TODO(T20329170): We might need this function in the Future if we decide + // to write in memory timestamps to overlay file on + // file handle close. } } - AbsolutePath FileInode::getLocalPath() const { return getMount()->getOverlay()->getFilePath(getNodeId()); } -std::pair> FileInode::isSameAsFast( - const Hash& blobID, - mode_t mode) { +folly::Optional FileInode::isSameAsFast(const Hash& blobID, mode_t mode) { // When comparing mode bits, we only care about the // file type and owner permissions. auto relevantModeBits = [](mode_t m) { return (m & (S_IFMT | S_IRWXU)); }; auto state = state_.wlock(); if (relevantModeBits(state->mode) != relevantModeBits(mode)) { - return std::make_pair(false, nullptr); + return false; } if (state->hash.hasValue()) { // This file is not materialized, so we can just compare hashes - return std::make_pair(state->hash.value() == blobID, nullptr); + return state->hash.value() == blobID; } - - return std::make_pair(false, getOrLoadData(state)); + return folly::none; } bool FileInode::isSameAs(const Blob& blob, mode_t mode) { auto result = isSameAsFast(blob.getHash(), mode); - if (!result.second) { - return result.first; + if (result.hasValue()) { + return result.value(); } - return result.second->getSha1() == Hash::sha1(&blob.getContents()); + return getSHA1().value() == Hash::sha1(&blob.getContents()); } folly::Future FileInode::isSameAs(const Hash& blobID, mode_t mode) { auto result = isSameAsFast(blobID, mode); - if (!result.second) { - return makeFuture(result.first); + if (result.hasValue()) { + return makeFuture(result.value()); } return getMount() ->getObjectStore() ->getBlobMetadata(blobID) .then([self = inodePtrFromThis()](const BlobMetadata& metadata) { - return self->getOrLoadData()->getSha1() == metadata.sha1; + return self->getSHA1().value() == metadata.sha1; }); } @@ -229,8 +209,6 @@ folly::Optional FileInode::getBlobHash() const { folly::Future> FileInode::open( const struct fuse_file_info& fi) { - shared_ptr data; - // TODO: We currently should ideally call fileHandleDidClose() if we fail // to create a FileHandle. It's currently slightly tricky to do this right // on all code paths. @@ -240,7 +218,6 @@ folly::Future> FileInode::open( // deprecated in future versions of C++. #if 0 SCOPE_EXIT { - data.reset(); fileHandleDidClose(); }; #endif @@ -257,22 +234,20 @@ folly::Future> FileInode::open( // punting on handling those situations here for now. throw InodeError(ELOOP, inodePtrFromThis(), "is a symlink"); } - - data = getOrLoadData(state); } if (fi.flags & (O_RDWR | O_WRONLY | O_CREAT | O_TRUNC)) { - return data->materializeForWrite(fi.flags).then( - [ self = inodePtrFromThis(), data, flags = fi.flags ]() { + return materializeForWrite(fi.flags).then( + [ self = inodePtrFromThis(), flags = fi.flags ]() { self->materializeInParent(); return shared_ptr{ - std::make_shared(self, data, flags)}; + std::make_shared(self, flags)}; }); } else { - return data->ensureDataLoaded().then( - [ self = inodePtrFromThis(), data, flags = fi.flags ]() { + return ensureDataLoaded().then( + [ self = inodePtrFromThis(), flags = fi.flags ]() { return shared_ptr{ - std::make_shared(self, data, flags)}; + std::make_shared(self, flags)}; }); } } @@ -286,12 +261,10 @@ void FileInode::materializeInParent() { } std::shared_ptr FileInode::finishCreate() { - auto data = getOrLoadData(); SCOPE_EXIT { - data.reset(); fileHandleDidClose(); }; - return std::make_shared(inodePtrFromThis(), data, 0); + return std::make_shared(inodePtrFromThis(), 0); } Future> FileInode::listxattr() { @@ -318,26 +291,448 @@ Future FileInode::getxattr(StringPiece name) { } Future FileInode::getSHA1(bool failIfSymlink) { - std::shared_ptr data; - folly::Optional hash; - { - auto state = state_.wlock(); - if (failIfSymlink && !S_ISREG(state->mode)) { - // We only define a SHA-1 value for regular files - return makeFuture(InodeError(kENOATTR, inodePtrFromThis())); - } - - hash = state->hash; - if (!hash.hasValue()) { - data = getOrLoadData(state); - } + auto state = state_.wlock(); + if (failIfSymlink && !S_ISREG(state->mode)) { + // We only define a SHA-1 value for regular files + return makeFuture(InodeError(kENOATTR, inodePtrFromThis())); } - if (hash.hasValue()) { - return getMount()->getObjectStore()->getSha1ForBlob(hash.value()); + if (state->hash.hasValue()) { + // If a file is not materialized it should have a hash value. + return getObjectStore()->getSha1ForBlob(state->hash.value()); + } else if (state->file) { + // If the file is materialized. + if (state->sha1Valid) { + auto shaStr = fgetxattr(state->file.fd(), kXattrSha1); + if (!shaStr.empty()) { + return Hash(shaStr); + } + } + return recomputeAndStoreSha1(state); + } else { + auto bug = EDEN_BUG() + << "One of state->hash and state->file must be set for the Inode :: " + << getNodeId() << " :Blob is " << (state->blob ? "not " : "") << "Null"; + return folly::makeFuture(bug.toException()); + } +} + +// Conditionally updates target with either the value provided by +// the caller, or with the current time value, depending on the value +// of the flags in to_set. Valid flag values are defined in fuse_lowlevel.h +// and have symbolic names matching FUSE_SET_*. +// useAttrFlag is the bitmask that indicates whether we should use the value +// from wantedTimeSpec. useNowFlag is the bitmask that indicates whether we +// should use the current time instead. +// If neither flag is present, we will preserve the current value in target. +static void resolveTimeForSetAttr( + struct timespec& target, + int to_set, + int useAttrFlag, + int useNowFlag, + const struct timespec& wantedTimeSpec) { + if (to_set & useAttrFlag) { + target = wantedTimeSpec; + } else if (to_set & useNowFlag) { + clock_gettime(CLOCK_REALTIME, &target); + } +} + +// Valid values for to_set are found in fuse_lowlevel.h and have symbolic +// names matching FUSE_SET_*. +struct stat FileInode::setAttr(const struct stat& attr, int to_set) { + auto state = state_.wlock(); + + CHECK(state->file) << "MUST have a materialized file at this point"; + + // We most likely need the current information to apply the requested + // changes below, so just fetch it here first. + struct stat currentStat; + checkUnixError(fstat(state->file.fd(), ¤tStat)); + + if (to_set & FUSE_SET_ATTR_SIZE) { + checkUnixError( + ftruncate(state->file.fd(), attr.st_size + Overlay::kHeaderLength)); } - return data->getSha1(); + if (to_set & (FUSE_SET_ATTR_UID | FUSE_SET_ATTR_GID)) { + if ((to_set & FUSE_SET_ATTR_UID && attr.st_uid != currentStat.st_uid) || + (to_set & FUSE_SET_ATTR_GID && attr.st_gid != currentStat.st_gid)) { + folly::throwSystemErrorExplicit( + EACCES, "changing the owner/group is not supported"); + } + + // Otherwise: there is no change + } + + if (to_set & FUSE_SET_ATTR_MODE) { + // The mode data is stored only in inode_->state_. + // (We don't set mode bits on the overlay file as that may incorrectly + // prevent us from reading or writing the overlay data). + // Make sure we preserve the file type bits, and only update permissions. + state->mode = (state->mode & S_IFMT) | (07777 & attr.st_mode); + } + + if (to_set & + (FUSE_SET_ATTR_ATIME | FUSE_SET_ATTR_MTIME | FUSE_SET_ATTR_ATIME_NOW | + FUSE_SET_ATTR_MTIME_NOW)) { + // Changing various time components. + // Element 0 is the atime, element 1 is the mtime. + struct timespec times[2] = {currentStat.st_atim, currentStat.st_mtim}; + + resolveTimeForSetAttr( + times[0], + to_set, + FUSE_SET_ATTR_ATIME, + FUSE_SET_ATTR_ATIME_NOW, + attr.st_atim); + + resolveTimeForSetAttr( + times[1], + to_set, + FUSE_SET_ATTR_MTIME, + FUSE_SET_ATTR_MTIME_NOW, + attr.st_mtim); + + checkUnixError(futimens(state->file.fd(), times)); + } + + // We need to return the now-current stat information for this file. + struct stat returnedStat; + checkUnixError(fstat(state->file.fd(), &returnedStat)); + returnedStat.st_mode = state->mode; + returnedStat.st_size -= Overlay::kHeaderLength; + + return returnedStat; +} + +struct stat FileInode::stat() { + auto st = getMount()->getMountPoint()->initStatData(); + st.st_nlink = 1; + + auto state = state_.rlock(); + + if (state->file) { + // stat() the overlay file. + // + // TODO: We need to get timestamps accurately here. + // The timestamps on the underlying file are not correct, because we keep + // file_ open for a long time, and do not close it when FUSE file handles + // close. (Timestamps are typically only updated on close operations.) + // This results our reported timestamps not changing correctly after the + // file is changed through FUSE APIs. + // + // We probably should update the overlay file to include a header, + // so we can store the atime, mtime, and ctime in the header data. + // Otherwise we won't be able to report the ctime accurately if we just + // keep using the overlay file timestamps. + checkUnixError(fstat(state->file.fd(), &st)); + + if (st.st_size < Overlay::kHeaderLength) { + auto filePath = getLocalPath(); + EDEN_BUG() << "Overlay file " << getLocalPath() + << " is too short for header: size=" << st.st_size; + } + + st.st_size -= Overlay::kHeaderLength; + st.st_mode = state->mode; + st.st_rdev = state->rdev; + + return st; + } + + CHECK(state->blob); + st.st_mode = state->mode; + + auto buf = state->blob->getContents(); + st.st_size = buf.computeChainDataLength(); + + // Report atime, mtime, and ctime as the time when we first loaded this + // FileInode. It hasn't been materialized yet, so this is a reasonble time + // to use. Once it is materialized we use the timestamps on the underlying + // overlay file, which the kernel keeps up-to-date. + auto epochTime = state->creationTime.time_since_epoch(); + auto epochSeconds = + std::chrono::duration_cast(epochTime); + st.st_atime = epochSeconds.count(); + st.st_mtime = epochSeconds.count(); + st.st_ctime = epochSeconds.count(); +#if defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || \ + _POSIX_C_SOURCE >= 200809L || _XOPEN_SOURCE >= 700 + auto nsec = std::chrono::duration_cast( + epochTime - epochSeconds); + st.st_atim.tv_nsec = nsec.count(); + st.st_mtim.tv_nsec = nsec.count(); + st.st_ctim.tv_nsec = nsec.count(); +#endif + + // NOTE: we don't set rdev to anything special here because we + // don't support committing special device nodes. + + return st; +} + +void FileInode::flush(uint64_t /* lock_owner */) { + // We have no write buffers, so there is nothing for us to flush, + // but let's take this opportunity to update the sha1 attribute. + auto state = state_.wlock(); + if (state->file && !state->sha1Valid) { + recomputeAndStoreSha1(state); + } +} + +void FileInode::fsync(bool datasync) { + auto state = state_.wlock(); + if (!state->file) { + // If we don't have an overlay file then we have nothing to sync. + return; + } + + auto res = +#ifndef __APPLE__ + datasync ? ::fdatasync(state->file.fd()) : +#endif + ::fsync(state->file.fd()); + checkUnixError(res); + + // let's take this opportunity to update the sha1 attribute. + if (!state->sha1Valid) { + recomputeAndStoreSha1(state); + } +} + +std::unique_ptr FileInode::readIntoBuffer( + size_t size, + off_t off) { + auto state = state_.rlock(); + + if (state->file) { + auto buf = folly::IOBuf::createCombined(size); + auto res = ::pread( + state->file.fd(), + buf->writableBuffer(), + size, + off + Overlay::kHeaderLength); + + checkUnixError(res); + buf->append(res); + return buf; + } + + auto buf = state->blob->getContents(); + folly::io::Cursor cursor(&buf); + + if (!cursor.canAdvance(off)) { + // Seek beyond EOF. Return an empty result. + return folly::IOBuf::wrapBuffer("", 0); + } + + cursor.skip(off); + + std::unique_ptr result; + cursor.cloneAtMost(result, size); + return result; +} + +std::string FileInode::readAll() { + auto state = state_.rlock(); + if (state->file) { + std::string result; + auto rc = lseek(state->file.fd(), Overlay::kHeaderLength, SEEK_SET); + folly::checkUnixError(rc, "unable to seek in materialized FileData"); + folly::readFile(state->file.fd(), result); + return result; + } + + const auto& contentsBuf = state->blob->getContents(); + folly::io::Cursor cursor(&contentsBuf); + return cursor.readFixedString(contentsBuf.computeChainDataLength()); +} + +fusell::BufVec FileInode::read(size_t size, off_t off) { + auto buf = readIntoBuffer(size, off); + return fusell::BufVec(std::move(buf)); +} + +size_t FileInode::write(fusell::BufVec&& buf, off_t off) { + auto state = state_.wlock(); + if (!state->file) { + // Not open for write + folly::throwSystemErrorExplicit(EINVAL); + } + + state->sha1Valid = false; + auto vec = buf.getIov(); + auto xfer = ::pwritev( + state->file.fd(), vec.data(), vec.size(), off + Overlay::kHeaderLength); + checkUnixError(xfer); + return xfer; +} + +size_t FileInode::write(folly::StringPiece data, off_t off) { + auto state = state_.wlock(); + if (!state->file) { + // Not open for write + folly::throwSystemErrorExplicit(EINVAL); + } + + state->sha1Valid = false; + auto xfer = ::pwrite( + state->file.fd(), data.data(), data.size(), off + Overlay::kHeaderLength); + checkUnixError(xfer); + return xfer; +} + +Future FileInode::ensureDataLoaded() { + auto state = state_.wlock(); + + if (!state->hash.hasValue()) { + // We should always have the file open if we are materialized. + CHECK(state->file); + return makeFuture(); + } + + if (state->blob) { + DCHECK_EQ(state->blob->getHash(), state->hash.value()); + return makeFuture(); + } + + // Load the blob data. + auto blobFuture = getObjectStore()->getBlob(state->hash.value()); + + // TODO: We really should defer this using a Future rather than calling get() + // here and blocking until the load completes. However, for that to work we + // will need to add some extra data tracking whether or not we are already in + // the process of loading the data. We need to avoid multiple threads all + // trying to load the data at the same time. + // + // For now doing a blocking load with the inode_->state_ lock held ensures + // that only one thread can load the data at a time. It's pretty unfortunate + // to block with the lock held, though :-( + state->blob = blobFuture.get(); + return makeFuture(); +} + +Future FileInode::materializeForWrite(int openFlags) { + auto state = state_.wlock(); + + // If we already have a materialized overlay file then we don't + // need to do much + if (state->file) { + CHECK(!state->hash.hasValue()); + if ((openFlags & O_TRUNC) != 0) { + // truncating a file that we already have open + state->sha1Valid = false; + checkUnixError(ftruncate(state->file.fd(), Overlay::kHeaderLength)); + auto emptySha1 = Hash::sha1(ByteRange{}); + storeSha1(state, emptySha1); + } else { + // no truncate option,overlay file contain old header + // we have to update only header but not contents + } + return makeFuture(); + } + + // Add header to the overlay File. + struct timespec zeroTime = {0, 0}; + auto header = Overlay::createHeader( + Overlay::kHeaderIdentifierFile, + Overlay::kHeaderVersion, + zeroTime, + zeroTime, + zeroTime); + auto iov = header.getIov(); + + // We must not be materialized yet + CHECK(state->hash.hasValue()); + + Hash sha1; + auto filePath = getLocalPath(); + + if ((openFlags & O_TRUNC) != 0) { + folly::writeFileAtomic(filePath.stringPiece(), iov.data(), iov.size()); + state->file = Overlay::openFile(filePath.stringPiece()); + sha1 = Hash::sha1(ByteRange{}); + } else { + if (!state->blob) { + // TODO: Load the blob using the non-blocking Future APIs. + // However, just as in ensureDataLoaded() above we will also need + // to add a mechanism to wait for already in-progress loads. + auto blobFuture = getObjectStore()->getBlob(state->hash.value()); + state->blob = blobFuture.get(); + } + + // Write the blob contents out to the overlay + auto contents = state->blob->getContents().getIov(); + iov.insert(iov.end(), contents.begin(), contents.end()); + + folly::writeFileAtomic( + filePath.stringPiece(), iov.data(), iov.size(), 0600); + state->file = Overlay::openFile(filePath.stringPiece()); + + sha1 = getObjectStore()->getSha1ForBlob(state->hash.value()); + } + + // Copy and apply the sha1 to the new file. This saves us from + // recomputing it again in the case that something opens the file + // read/write and closes it without changing it. + storeSha1(state, sha1); + + // Update the FileInode to indicate that we are materialized now + state->blob.reset(); + state->hash = folly::none; + + return makeFuture(); +} + +ObjectStore* FileInode::getObjectStore() const { + return getMount()->getObjectStore(); +} + +Hash FileInode::recomputeAndStoreSha1( + const folly::Synchronized::LockedPtr& state) { + uint8_t buf[8192]; + off_t off = Overlay::kHeaderLength; + SHA_CTX ctx; + SHA1_Init(&ctx); + + while (true) { + // Using pread here so that we don't move the file position; + // the file descriptor is shared between multiple file handles + // and while we serialize the requests to FileData, it seems + // like a good property of this function to avoid changing that + // state. + auto len = folly::preadNoInt(state->file.fd(), buf, sizeof(buf), off); + if (len == 0) { + break; + } + if (len == -1) { + folly::throwSystemError(); + } + SHA1_Update(&ctx, buf, len); + off += len; + } + + uint8_t digest[SHA_DIGEST_LENGTH]; + SHA1_Final(digest, &ctx); + auto sha1 = Hash(folly::ByteRange(digest, sizeof(digest))); + storeSha1(state, sha1); + return sha1; +} + +void FileInode::storeSha1( + const folly::Synchronized::LockedPtr& state, + Hash sha1) { + try { + fsetxattr(state->file.fd(), kXattrSha1, sha1.toString()); + state->sha1Valid = true; + } catch (const std::exception& ex) { + // If something goes wrong storing the attribute just log a warning + // and leave sha1Valid as false. We'll have to recompute the value + // next time we need it. + XLOG(WARNING) << "error setting SHA1 attribute in the overlay: " + << folly::exceptionStr(ex); + } } } } diff --git a/eden/fs/inodes/FileInode.h b/eden/fs/inodes/FileInode.h index fb0f6b5a12..e652b75985 100644 --- a/eden/fs/inodes/FileInode.h +++ b/eden/fs/inodes/FileInode.h @@ -22,10 +22,15 @@ class File; namespace facebook { namespace eden { +namespace fusell { +class BufVec; +} + class Blob; class FileHandle; class FileData; class Hash; +class ObjectStore; class FileInode : public InodeBase { public: @@ -69,9 +74,6 @@ class FileInode : public InodeBase { folly::Future getxattr(folly::StringPiece name) override; folly::Future getSHA1(bool failIfSymlink = true); - /// Ensure that underlying storage information is loaded - std::shared_ptr getOrLoadData(); - /// Compute the path to the overlay file for this item. AbsolutePath getLocalPath() const; @@ -114,6 +116,48 @@ class FileInode : public InodeBase { */ folly::Optional getBlobHash() const; + /** + * Read the entire file contents, and return them as a string. + * + * Note that this API generally should only be used for fairly small files. + */ + std::string readAll(); + + /** + * Load the file data so it can be used for reading. + * + * If this file is materialized, this opens its file in the overlay. + * If the file is not materialized, this loads the Blob data from the + * ObjectStore. + */ + FOLLY_NODISCARD folly::Future ensureDataLoaded(); + + /** + * Materialize the file data. + * openFlags has the same meaning as the flags parameter to + * open(2). Materialization depends on the write mode specified + * in those flags; if we are writing to the file then we need to + * copy it locally to the overlay. If we are truncating we just + * need to create an empty file in the overlay. Otherwise we + * need to go out to the LocalStore to obtain the backing data. + */ + FOLLY_NODISCARD folly::Future materializeForWrite(int openFlags); + + /** + * Read up to size bytes from the file at the specified offset. + * + * Returns an IOBuf containing the data. This may return fewer bytes than + * requested. If the specified offset is at or past the end of the buffer an + * empty IOBuf will be returned. Otherwise between 1 and size bytes will be + * returned. If fewer than size bytes are returned this does *not* guarantee + * that the end of the file was reached. + * + * May throw exceptions on error. + */ + std::unique_ptr readIntoBuffer(size_t size, off_t off); + + size_t write(folly::StringPiece data, off_t off); + private: /** * The contents of a FileInode. @@ -127,7 +171,6 @@ class FileInode : public InodeBase { State(FileInode* inode, mode_t mode, folly::File&& hash, dev_t rdev = 0); ~State(); - std::shared_ptr data; mode_t mode{0}; dev_t rdev{0}; /** @@ -137,16 +180,18 @@ class FileInode : public InodeBase { folly::Optional hash; /** - * data members from FileData + * If backed by tree, the data from the tree, else nullptr. */ - - /// if backed by tree, the data from the tree, else nullptr. std::unique_ptr blob; - /// if backed by an overlay file, whether the sha1 xattr is valid + /** + * If backed by an overlay file, whether the sha1 xattr is valid + */ bool sha1Valid{false}; - /// if backed by an overlay file, the open file descriptor + /** + * If backed by an overlay file, the open file descriptor. + */ folly::File file; }; @@ -162,8 +207,6 @@ class FileInode : public InodeBase { FileInodePtr inodePtrFromThis() { return FileInodePtr::newPtrFromExisting(this); } - std::shared_ptr getOrLoadData( - const folly::Synchronized::LockedPtr& state); /** * Mark this FileInode materialized in its parent directory. @@ -179,12 +222,36 @@ class FileInode : public InodeBase { * Helper function for isSameAs(). * * This does the initial portion of the check which never requires a Future. - * Returns (bool, nullptr) if the check completes immediately, or - * (false, data) if the contents need to be checked against the FileData. + * Returns Optional if the check completes immediately, or + * folly::none if the contents need to be checked against sha1 of file + * contents. */ - std::pair> isSameAsFast( - const Hash& blobID, - mode_t mode); + folly::Optional isSameAsFast(const Hash& blobID, mode_t mode); + + /** + * Change attributes for this inode. + * attr is a standard struct stat. Only the members indicated + * by to_set are valid. Defined values for the to_set flags + * are found in the fuse_lowlevel.h header file and have symbolic + * names matching FUSE_SET_*. + */ + struct stat setAttr(const struct stat& attr, int to_set); + + /** + * Recompute the SHA1 content hash of the open file. + */ + Hash recomputeAndStoreSha1( + const folly::Synchronized::LockedPtr& state); + + ObjectStore* getObjectStore() const; + void storeSha1( + const folly::Synchronized::LockedPtr& state, + Hash sha1); + fusell::BufVec read(size_t size, off_t off); + size_t write(fusell::BufVec&& buf, off_t off); + struct stat stat(); + void flush(uint64_t lock_owner); + void fsync(bool datasync); folly::Synchronized state_; diff --git a/eden/fs/inodes/TreeInode.cpp b/eden/fs/inodes/TreeInode.cpp index 42bf80eaa8..2544bc40af 100644 --- a/eden/fs/inodes/TreeInode.cpp +++ b/eden/fs/inodes/TreeInode.cpp @@ -24,7 +24,6 @@ #include "eden/fs/inodes/DiffContext.h" #include "eden/fs/inodes/EdenDispatcher.h" #include "eden/fs/inodes/EdenMount.h" -#include "eden/fs/inodes/FileData.h" #include "eden/fs/inodes/FileHandle.h" #include "eden/fs/inodes/FileInode.h" #include "eden/fs/inodes/InodeDiffCallback.h" @@ -1621,18 +1620,16 @@ Future TreeInode::loadGitIgnoreThenDiff( isIgnored); } - auto data = fileInode->getOrLoadData(); if (S_ISLNK(fileInode->getMode())) { - auto dataFuture = data->ensureDataLoaded(); - return dataFuture.then( - [ fileInode = std::move(fileInode), data = std::move(data) ]() { - // auto symlinkContents = data->readAll(); - // TODO: Look up the symlink destination and continue. - // The symlink might point to another path inside our mount point, or - // it may point outside. - return makeFuture(std::runtime_error( - "handling .gitignore symlinks not implemented yet")); - }); + auto dataFuture = fileInode->ensureDataLoaded(); + return dataFuture.then([fileInode = std::move(fileInode)]() { + // auto symlinkContents = data->readAll(); + // TODO: Look up the symlink destination and continue. + // The symlink might point to another path inside our mount point, or + // it may point outside. + return makeFuture(std::runtime_error( + "handling .gitignore symlinks not implemented yet")); + }); } // Load the file data @@ -1641,7 +1638,7 @@ Future TreeInode::loadGitIgnoreThenDiff( // move-captures. Before C++17 the ordering is undefined here, and the // compiler may decide to move data away before evaluating // data->ensureDataLoaded(). - auto dataFuture = data->ensureDataLoaded(); + auto dataFuture = fileInode->ensureDataLoaded(); return dataFuture.then([ self = inodePtrFromThis(), context, @@ -1649,9 +1646,9 @@ Future TreeInode::loadGitIgnoreThenDiff( tree = std::move(tree), parentIgnore, isIgnored, - data = std::move(data) + fileInode = std::move(fileInode) ]() mutable { - auto ignoreFileContents = data->readAll(); + auto ignoreFileContents = fileInode->readAll(); auto ignore = make_unique(parentIgnore, ignoreFileContents); return self->computeDiff( self->contents_.wlock(), diff --git a/eden/fs/inodes/test/RemoveTest.cpp b/eden/fs/inodes/test/RemoveTest.cpp index 93ba693d36..041dd59fe9 100644 --- a/eden/fs/inodes/test/RemoveTest.cpp +++ b/eden/fs/inodes/test/RemoveTest.cpp @@ -9,7 +9,6 @@ */ #include -#include "eden/fs/inodes/FileData.h" #include "eden/fs/inodes/FileInode.h" #include "eden/fs/inodes/TreeInode.h" #include "eden/fs/testharness/FakeTreeBuilder.h" @@ -93,14 +92,13 @@ TEST_F(UnlinkTest, modified) { // Modify the child, so it is materialized before we remove it auto file = mount_.getFileInode("dir/a.txt"); EXPECT_EQ(file->getNodeId(), dir->getChildInodeNumber(childPath)); - auto fileData = file->getOrLoadData(); - fileData->materializeForWrite(O_WRONLY).get(); + file->materializeForWrite(O_WRONLY).get(); auto newContents = StringPiece{ "new contents for the file\n" "testing testing\n" "123\n" "testing testing\n"}; - auto writeResult = fileData->write(newContents, 0); + auto writeResult = file->write(newContents, 0); EXPECT_EQ(newContents.size(), writeResult); // Now remove the child diff --git a/eden/fs/testharness/TestChecks.h b/eden/fs/testharness/TestChecks.h index c992c287fa..afb9624c47 100644 --- a/eden/fs/testharness/TestChecks.h +++ b/eden/fs/testharness/TestChecks.h @@ -13,21 +13,18 @@ #include #include #include -#include "eden/fs/inodes/FileData.h" #include "eden/fs/inodes/FileInode.h" /** * Check that a FileInode has the expected contents and permissions. */ -#define EXPECT_FILE_INODE(fileInode, expectedData, expectedPerms) \ - do { \ - auto fileDataForCheck = (fileInode)->getOrLoadData(); \ - auto loadFuture = fileDataForCheck->ensureDataLoaded(); \ - ASSERT_TRUE(loadFuture.isReady()); \ - loadFuture.get(); \ - EXPECT_EQ( \ - (expectedData), folly::StringPiece{fileDataForCheck->readAll()}); \ - EXPECT_EQ( \ - folly::sformat("{:#o}", (expectedPerms)), \ - folly::sformat("{:#o}", (fileInode)->getPermissions())); \ +#define EXPECT_FILE_INODE(fileInode, expectedData, expectedPerms) \ + do { \ + auto loadFuture = fileInode->ensureDataLoaded(); \ + ASSERT_TRUE(loadFuture.isReady()); \ + loadFuture.get(); \ + EXPECT_EQ(expectedData, folly::StringPiece{fileInode->readAll()}); \ + EXPECT_EQ( \ + folly::sformat("{:#o}", (expectedPerms)), \ + folly::sformat("{:#o}", (fileInode)->getPermissions())); \ } while (0) diff --git a/eden/fs/testharness/TestMount.cpp b/eden/fs/testharness/TestMount.cpp index 56febe4ae7..2314893b23 100644 --- a/eden/fs/testharness/TestMount.cpp +++ b/eden/fs/testharness/TestMount.cpp @@ -17,7 +17,6 @@ #include "eden/fs/fuse/MountPoint.h" #include "eden/fs/inodes/Dirstate.h" #include "eden/fs/inodes/EdenDispatcher.h" -#include "eden/fs/inodes/FileData.h" #include "eden/fs/inodes/FileHandle.h" #include "eden/fs/inodes/FileInode.h" #include "eden/fs/inodes/Overlay.h" @@ -260,9 +259,8 @@ void TestMount::overwriteFile(folly::StringPiece path, std::string contents) { std::string TestMount::readFile(folly::StringPiece path) { auto file = getFileInode(path); - auto fileData = file->getOrLoadData(); auto attr = file->getattr().get(); - auto buf = fileData->readIntoBuffer( + auto buf = file->readIntoBuffer( /* size */ attr.st.st_size, /* off */ 0); return std::string(reinterpret_cast(buf->data()), buf->length()); }