facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-06 14:58:03 +03:00
Code Issues Projects Releases Wiki Activity

overlay header file related changes

Browse Source 
Summary: Added header to the files that are materialized .

Reviewed By: simpkins

Differential Revision: D5387990

fbshipit-source-id: 1d551d674a39e01d6314d8bb3308e7bea3e669fc
This commit is contained in:
Jyothsna Konisa 2017-07-13 17:11:36 -07:00 committed by Facebook Github Bot
parent 128787b01b
commit 981d1d660a
5 changed files with 153 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
eden/fs/inodes/FileData.cpp
View File

@ -28,6 +28,7 @@
#include "eden/fs/model/Blob.h"
#include "eden/fs/model/Hash.h"
#include "eden/fs/store/ObjectStore.h"
#include "eden/fs/utils/Bug.h"
#include "eden/fs/utils/XAttr.h"
using folly::ByteRange;
@ -35,6 +36,7 @@ using folly::checkUnixError;
using folly::Future;
using folly::makeFuture;
using folly::Unit;
using folly::StringPiece;
namespace facebook {
namespace eden {
@ -45,7 +47,7 @@ FileData::FileData(FileInode* inode, const folly::Optional<Hash>& hash)
// this is a materialized file.
if (!hash.hasValue()) {
auto filePath = inode_->getLocalPath();
file_ = folly::File(filePath.c_str(), O_RDWR | O_NOFOLLOW, 0600);
file_ = Overlay::openFile(filePath.c_str());
}
}
@ -86,7 +88,8 @@ struct stat FileData::setAttr(const struct stat& attr, int to_set) {
checkUnixError(fstat(file_.fd(), &currentStat));
if (to_set & FUSE_SET_ATTR_SIZE) {
checkUnixError(ftruncate(file_.fd(), attr.st_size));
checkUnixError(
ftruncate(file_.fd(), attr.st_size + Overlay::kHeaderLength));
}
if (to_set & (FUSE_SET_ATTR_UID | FUSE_SET_ATTR_GID)) {
@ -134,6 +137,7 @@ struct stat FileData::setAttr(const struct stat& attr, int to_set) {
struct stat returnedStat;
checkUnixError(fstat(file_.fd(), &returnedStat));
returnedStat.st_mode = state->mode;
returnedStat.st_size -= Overlay::kHeaderLength;
return returnedStat;
}
@ -159,8 +163,17 @@ struct stat FileData::stat() {
// Otherwise we won't be able to report the ctime accurately if we just
// keep using the overlay file timestamps.
checkUnixError(fstat(file_.fd(), &st));
if (st.st_size < Overlay::kHeaderLength) {
auto filePath = inode_->getLocalPath();
EDEN_BUG() << "Overlay file " << inode_->getLocalPath()
<< " is too short for header: size=" << st.st_size;
}
st.st_size -= Overlay::kHeaderLength;
st.st_mode = state->mode;
st.st_rdev = state->rdev;
return st;
}
@ -229,7 +242,9 @@ std::unique_ptr<folly::IOBuf> FileData::readIntoBuffer(size_t size, off_t off) {
if (file_) {
auto buf = folly::IOBuf::createCombined(size);
auto res = ::pread(file_.fd(), buf->writableBuffer(), size, off);
auto res = ::pread(
file_.fd(), buf->writableBuffer(), size, off + Overlay::kHeaderLength);
checkUnixError(res);
buf->append(res);
return buf;
@ -254,7 +269,7 @@ std::string FileData::readAll() {
auto state = inode_->state_.rlock();
if (file_) {
std::string result;
auto rc = lseek(file_.fd(), 0, SEEK_SET);
auto rc = lseek(file_.fd(), Overlay::kHeaderLength, SEEK_SET);
folly::checkUnixError(rc, "unable to seek in materialized FileData");
folly::readFile(file_.fd(), result);
return result;
@ -279,7 +294,8 @@ size_t FileData::write(fusell::BufVec&& buf, off_t off) {
sha1Valid_ = false;
auto vec = buf.getIov();
auto xfer = ::pwritev(file_.fd(), vec.data(), vec.size(), off);
auto xfer = ::pwritev(
file_.fd(), vec.data(), vec.size(), off + Overlay::kHeaderLength);
checkUnixError(xfer);
return xfer;
}
@ -292,7 +308,8 @@ size_t FileData::write(folly::StringPiece data, off_t off) {
}
sha1Valid_ = false;
auto xfer = ::pwrite(file_.fd(), data.data(), data.size(), off);
auto xfer = ::pwrite(
file_.fd(), data.data(), data.size(), off + Overlay::kHeaderLength);
checkUnixError(xfer);
return xfer;
}
@ -337,20 +354,35 @@ Future<Unit> FileData::materializeForWrite(int openFlags) {
if ((openFlags & O_TRUNC) != 0) {
// truncating a file that we already have open
sha1Valid_ = false;
checkUnixError(ftruncate(file_.fd(), 0));
checkUnixError(ftruncate(file_.fd(), Overlay::kHeaderLength));
auto emptySha1 = Hash::sha1(ByteRange{});
storeSha1(state, emptySha1);
} else {
// no truncate option,overlay file contain old header
// we have to update only header but not contents
}
return makeFuture();
}
// Add header to the overlay File.
struct timespec zeroTime = {0, 0};
auto header = Overlay::createHeader(
Overlay::kHeaderIdentifierFile,
Overlay::kHeaderVersion,
zeroTime,
zeroTime,
zeroTime);
auto iov = header.getIov();
// We must not be materialized yet
CHECK(state->hash.hasValue());
Hash sha1;
auto filePath = inode_->getLocalPath();
if ((openFlags & O_TRUNC) != 0) {
file_ = folly::File(filePath.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0600);
folly::writeFileAtomic(filePath.stringPiece(), iov.data(), iov.size());
file_ = Overlay::openFile(filePath.stringPiece());
sha1 = Hash::sha1(ByteRange{});
} else {
if (!blob_) {
@ -362,10 +394,12 @@ Future<Unit> FileData::materializeForWrite(int openFlags) {
}
// Write the blob contents out to the overlay
auto iov = blob_->getContents().getIov();
auto contents = blob_->getContents().getIov();
iov.insert(iov.end(), contents.begin(), contents.end());
folly::writeFileAtomic(
filePath.stringPiece(), iov.data(), iov.size(), 0600);
file_ = folly::File(filePath.c_str(), O_RDWR);
file_ = Overlay::openFile(filePath.stringPiece());
sha1 = getObjectStore()->getSha1ForBlob(state->hash.value());
}
@ -407,7 +441,7 @@ ObjectStore* FileData::getObjectStore() const {
Hash FileData::recomputeAndStoreSha1(
const folly::Synchronized<FileInode::State>::LockedPtr& state) {
uint8_t buf[8192];
off_t off = 0;
off_t off = Overlay::kHeaderLength;
SHA_CTX ctx;
SHA1_Init(&ctx);

85
eden/fs/inodes/Overlay.cpp
View File

@ -39,9 +39,7 @@ using std::unique_ptr;
constexpr StringPiece kMetaDir{"overlay"};
constexpr StringPiece kMetaFile{"dirdata"};
constexpr StringPiece kInfoFile{"info"};
constexpr StringPiece kHeaderIdentifier{"OVDR"};
constexpr uint32_t kHeaderVersion = 1;
constexpr size_t kHeaderLength = 64;
/**
* 4-byte magic identifier to put at the start of the info file.
* This merely helps confirm that we are in fact reading an overlay info file
@ -81,6 +79,11 @@ void formatSubdirPath(MutableStringPiece subdirPath, fuse_ino_t inode) {
}
}
constexpr folly::StringPiece Overlay::kHeaderIdentifierDir;
constexpr folly::StringPiece Overlay::kHeaderIdentifierFile;
constexpr uint32_t Overlay::kHeaderVersion;
constexpr size_t Overlay::kHeaderLength;
Overlay::Overlay(AbsolutePathPiece localDir) : localDir_(localDir) {
initOverlay();
}
@ -261,7 +264,7 @@ void Overlay::saveOverlayDir(
struct timespec zeroTime = {0, 0};
auto header = createHeader(
kHeaderIdentifier, kHeaderVersion, zeroTime, zeroTime, zeroTime);
kHeaderIdentifierDir, kHeaderVersion, zeroTime, zeroTime, zeroTime);
auto iov = header.getIov();
iov.push_back(
@ -388,8 +391,8 @@ Optional<overlay::OverlayDir> Overlay::deserializeOverlayDir(
contents.advance(kHeaderLength);
// check if the header contains valid identifier and version
StringPiece identifier{header, 0, kHeaderIdentifier.size()};
header.advance(kHeaderIdentifier.size());
StringPiece identifier{header, 0, kHeaderIdentifierDir.size()};
header.advance(kHeaderIdentifierDir.size());
StringPiece version{header, 0, sizeof(kHeaderVersion)};
folly::IOBuf buf(folly::IOBuf::WRAP_BUFFER, ByteRange{version});
@ -397,7 +400,7 @@ Optional<overlay::OverlayDir> Overlay::deserializeOverlayDir(
auto ver = cursor.readBE<uint32_t>();
// Header doesn't contain identifier
if (identifier.compare(kHeaderIdentifier) != 0) {
if (identifier.compare(kHeaderIdentifierDir) != 0) {
folly::throwSystemError(
EIO,
"unexpected overlay header identifier in ",
@ -439,5 +442,73 @@ folly::IOBuf Overlay::createHeader(
return header;
}
// Helper function to open,validate,
// get file pointer of an overlay file
folly::File Overlay::openFile(folly::StringPiece filePath) {
// Open the overlay file
folly::File file(filePath, O_RDWR);
// Read the contents
std::string contents;
folly::readFile(file.fd(), contents);
StringPiece header{contents, 0, kHeaderLength};
StringPiece identifier{header, 0, kHeaderIdentifierFile.size()};
header.advance(kHeaderIdentifierFile.size());
StringPiece version{header, 0, sizeof(kHeaderVersion)};
folly::IOBuf buf(folly::IOBuf::WRAP_BUFFER, ByteRange{version});
folly::io::Cursor cursor(&buf);
auto ver = cursor.readBE<uint32_t>();
// Header doesn't contain identifier
if (identifier.compare(kHeaderIdentifierFile) != 0) {
folly::throwSystemError(
EIO,
"unexpected overlay header identifier in ",
filePath,
": ",
folly::hexlify(ByteRange{identifier}));
}
// Version number is different
if (ver != kHeaderVersion) {
folly::throwSystemError(
EIO, "Unexpected overlay version ", ver, " in ", filePath);
}
return file;
}
// Helper function to add header to the materialized file
void Overlay::addHeaderToOverlayFile(int fd) {
struct timespec zeroTime = {0, 0};
auto header = createHeader(
kHeaderIdentifierFile, kHeaderVersion, zeroTime, zeroTime, zeroTime);
auto data = header.coalesce();
auto wrote = folly::writeFull(fd, data.data(), data.size());
if (wrote == -1) {
folly::throwSystemError("writeNoInt failed");
}
if (wrote != data.size()) {
folly::throwSystemError(
"writeNoInt wrote only ", wrote, " of ", data.size(), " bytes");
}
}
// Helper function to create an overlay file
folly::File Overlay::createOverlayFile(fuse_ino_t childNumber) {
auto filePath = getFilePath(childNumber);
folly::File file(filePath.c_str(), O_RDWR | O_CREAT | O_EXCL, 0600);
SCOPE_FAIL {
::unlink(filePath.c_str());
};
addHeaderToOverlayFile(file.fd());
return file;
}
}
}

23
eden/fs/inodes/Overlay.h
View File

@ -67,6 +67,17 @@ class Overlay {
const struct timespec& ctime,
const struct timespec& mtime);
/**
* Helper function that opens an existing overlay file,
* checks that it is valid, and returns the file.
*/
static folly::File openFile(folly::StringPiece filePath);
/**
* Helper function that creates a new overlay file and adds header to it
*/
folly::File createOverlayFile(fuse_ino_t childNumber);
/**
* Get the maximum inode number stored in the overlay.
*
@ -76,6 +87,14 @@ class Overlay {
*/
fuse_ino_t getMaxRecordedInode();
/**
* Constants for an header in overlay file.
*/
static constexpr folly::StringPiece kHeaderIdentifierDir{"OVDR"};
static constexpr folly::StringPiece kHeaderIdentifierFile{"OVFL"};
static constexpr uint32_t kHeaderVersion = 1;
static constexpr size_t kHeaderLength = 64;
private:
void initOverlay();
bool isOldFormatOverlay() const;
@ -83,6 +102,10 @@ class Overlay {
void initNewOverlay();
folly::Optional<overlay::OverlayDir> deserializeOverlayDir(
fuse_ino_t inodeNumber) const;
/**
* Helper function to add header to the overlay file
*/
static void addHeaderToOverlayFile(int fd);
/** path to ".eden/CLIENT/local" */
AbsolutePath localDir_;

22
eden/fs/inodes/TreeInode.cpp
View File

@ -670,12 +670,8 @@ TreeInode::create(PathComponentPiece name, mode_t mode, int flags) {
// Since we will move this file into the underlying file data, we
// take special care to ensure that it is opened read-write
auto filePath = getOverlay()->getFilePath(childNumber);
folly::File file(
filePath.c_str(),
O_RDWR | O_CREAT | (flags & ~(O_RDONLY | O_WRONLY)),
0600);
folly::File file = getOverlay()->createOverlayFile(childNumber);
// The mode passed in by the caller may not have the file type bits set.
// Ensure that we mark this as a regular file.
mode = S_IFREG | (07777 & mode);
@ -751,18 +747,17 @@ FileInodePtr TreeInode::symlink(
auto* inodeMap = this->getInodeMap();
auto childNumber = inodeMap->allocateInodeNumber();
auto filePath = getOverlay()->getFilePath(childNumber);
folly::File file = getOverlay()->createOverlayFile(childNumber);
folly::File file(filePath.c_str(), O_RDWR | O_CREAT | O_EXCL, 0600);
SCOPE_FAIL {
::unlink(filePath.c_str());
};
auto wrote = folly::writeNoInt(
file.fd(), symlinkTarget.data(), symlinkTarget.size());
if (wrote == -1) {
auto filePath = getOverlay()->getFilePath(childNumber);
folly::throwSystemError("writeNoInt(", filePath, ") failed");
}
if (wrote != symlinkTarget.size()) {
auto filePath = getOverlay()->getFilePath(childNumber);
folly::throwSystemError(
"writeNoInt(",
filePath,
@ -837,12 +832,7 @@ TreeInode::mknod(PathComponentPiece name, mode_t mode, dev_t rdev) {
auto* inodeMap = this->getInodeMap();
auto childNumber = inodeMap->allocateInodeNumber();
auto filePath = getOverlay()->getFilePath(childNumber);
folly::File file(filePath.c_str(), O_RDWR | O_CREAT | O_EXCL, 0600);
SCOPE_FAIL {
::unlink(filePath.c_str());
};
folly::File file = getOverlay()->createOverlayFile(childNumber);
auto entry = std::make_unique<Entry>(mode, childNumber, rdev);
// build a corresponding FileInode

1
eden/fs/inodes/TreeInode.h
View File

@ -8,6 +8,7 @@
*
*/
#pragma once
#include <folly/File.h>
#include <folly/Optional.h>
#include <folly/Portability.h>
#include <folly/Synchronized.h>