2016-09-10 02:56:02 +03:00
|
|
|
/*
|
2017-01-21 09:02:33 +03:00
|
|
|
* Copyright (c) 2016-present, Facebook, Inc.
|
2016-09-10 02:56:02 +03:00
|
|
|
* All rights reserved.
|
|
|
|
*
|
|
|
|
* This source code is licensed under the BSD-style license found in the
|
|
|
|
* LICENSE file in the root directory of this source tree. An additional grant
|
|
|
|
* of patent rights can be found in the PATENTS file in the same directory.
|
|
|
|
*
|
|
|
|
*/
|
2018-03-29 06:42:21 +03:00
|
|
|
#include "eden/fs/inodes/Overlay.h"
|
|
|
|
|
2017-02-11 01:16:00 +03:00
|
|
|
#include <boost/filesystem.hpp>
|
2016-09-10 02:56:02 +03:00
|
|
|
#include <folly/Exception.h>
|
2017-02-11 01:16:00 +03:00
|
|
|
#include <folly/File.h>
|
2016-09-10 02:56:02 +03:00
|
|
|
#include <folly/FileUtil.h>
|
2018-03-24 04:17:05 +03:00
|
|
|
#include <folly/experimental/logging/xlog.h>
|
2017-07-04 10:18:17 +03:00
|
|
|
#include <folly/io/Cursor.h>
|
|
|
|
#include <folly/io/IOBuf.h>
|
2016-09-10 02:56:02 +03:00
|
|
|
#include <thrift/lib/cpp2/protocol/Serializer.h>
|
2018-03-08 09:57:38 +03:00
|
|
|
#include "eden/fs/inodes/InodeMap.h"
|
2016-09-10 02:56:02 +03:00
|
|
|
#include "eden/fs/inodes/gen-cpp2/overlay_types.h"
|
2017-04-14 21:31:48 +03:00
|
|
|
#include "eden/fs/utils/PathFuncs.h"
|
2016-09-10 02:56:02 +03:00
|
|
|
|
|
|
|
namespace facebook {
|
|
|
|
namespace eden {
|
|
|
|
|
2017-02-11 01:16:00 +03:00
|
|
|
using apache::thrift::CompactSerializer;
|
|
|
|
using folly::ByteRange;
|
2018-01-26 22:17:36 +03:00
|
|
|
using folly::fbvector;
|
2017-02-11 01:16:00 +03:00
|
|
|
using folly::File;
|
2018-03-28 22:54:24 +03:00
|
|
|
using folly::IOBuf;
|
2017-02-11 01:16:00 +03:00
|
|
|
using folly::MutableStringPiece;
|
|
|
|
using folly::Optional;
|
|
|
|
using folly::StringPiece;
|
2018-03-29 06:42:21 +03:00
|
|
|
using folly::literals::string_piece_literals::operator""_sp;
|
2017-02-11 01:16:00 +03:00
|
|
|
using std::string;
|
2016-09-10 02:56:02 +03:00
|
|
|
|
|
|
|
/* Relative to the localDir, the metaFile holds the serialized rendition
|
|
|
|
* of the overlay_ data. We use thrift CompactSerialization for this.
|
|
|
|
*/
|
|
|
|
constexpr StringPiece kMetaDir{"overlay"};
|
|
|
|
constexpr StringPiece kMetaFile{"dirdata"};
|
2017-02-11 01:16:00 +03:00
|
|
|
constexpr StringPiece kInfoFile{"info"};
|
2017-07-14 03:11:36 +03:00
|
|
|
|
2017-02-11 01:16:00 +03:00
|
|
|
/**
|
|
|
|
* 4-byte magic identifier to put at the start of the info file.
|
|
|
|
* This merely helps confirm that we are in fact reading an overlay info file
|
|
|
|
*/
|
|
|
|
constexpr StringPiece kInfoHeaderMagic{"\xed\xe0\x00\x01"};
|
|
|
|
|
|
|
|
/**
|
|
|
|
* A version number for the overlay directory format.
|
|
|
|
*
|
|
|
|
* If we change the overlay storage format in the future we can bump this
|
|
|
|
* version number to help identify when eden is reading overlay data created by
|
|
|
|
* an older version of the code.
|
|
|
|
*/
|
|
|
|
constexpr uint32_t kOverlayVersion = 1;
|
|
|
|
constexpr size_t kInfoHeaderSize =
|
|
|
|
kInfoHeaderMagic.size() + sizeof(kOverlayVersion);
|
2016-09-10 02:56:02 +03:00
|
|
|
|
|
|
|
/* Relative to the localDir, the overlay tree is where we create the
|
|
|
|
* materialized directory structure; directories and files are created
|
|
|
|
* here. */
|
|
|
|
constexpr StringPiece kOverlayTree{"tree"};
|
|
|
|
|
2017-02-11 01:16:00 +03:00
|
|
|
namespace {
|
|
|
|
/**
|
|
|
|
* Get the name of the subdirectory to use for the overlay data for the
|
|
|
|
* specified inode number.
|
|
|
|
*
|
|
|
|
* We shard the inode files across the 256 subdirectories using the least
|
|
|
|
* significant byte. Inode numbers are allocated in monotonically increasing
|
|
|
|
* order, so this helps spread them out across the subdirectories.
|
|
|
|
*/
|
2018-02-27 23:40:30 +03:00
|
|
|
void formatSubdirPath(MutableStringPiece subdirPath, uint64_t inode) {
|
2017-02-11 01:16:00 +03:00
|
|
|
constexpr char hexdigit[] = "0123456789abcdef";
|
|
|
|
DCHECK_EQ(subdirPath.size(), 2);
|
|
|
|
subdirPath[0] = hexdigit[(inode >> 4) & 0xf];
|
|
|
|
subdirPath[1] = hexdigit[inode & 0xf];
|
|
|
|
}
|
2017-11-04 01:58:04 +03:00
|
|
|
} // namespace
|
2017-02-11 01:16:00 +03:00
|
|
|
|
2017-07-14 03:11:36 +03:00
|
|
|
constexpr folly::StringPiece Overlay::kHeaderIdentifierDir;
|
|
|
|
constexpr folly::StringPiece Overlay::kHeaderIdentifierFile;
|
|
|
|
constexpr uint32_t Overlay::kHeaderVersion;
|
|
|
|
constexpr size_t Overlay::kHeaderLength;
|
|
|
|
|
2017-02-11 01:16:00 +03:00
|
|
|
Overlay::Overlay(AbsolutePathPiece localDir) : localDir_(localDir) {
|
|
|
|
initOverlay();
|
|
|
|
}
|
|
|
|
|
|
|
|
void Overlay::initOverlay() {
|
|
|
|
// Read the overlay version file. If it does not exist, create it.
|
|
|
|
//
|
|
|
|
// First check for an old-format overlay directory, before we wrote out
|
|
|
|
// version numbers. This is only to warn developers if they try to use
|
|
|
|
// eden with an existing older client. We can probably delete this check in
|
|
|
|
// a few weeks.
|
|
|
|
if (isOldFormatOverlay()) {
|
|
|
|
throw std::runtime_error(
|
|
|
|
"The eden overlay format has been upgraded. "
|
|
|
|
"This version of eden cannot use the old overlay directory at " +
|
2018-01-04 03:31:08 +03:00
|
|
|
localDir_.value());
|
2017-02-11 01:16:00 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
// Read the info file.
|
|
|
|
auto infoPath = localDir_ + PathComponentPiece{kInfoFile};
|
2018-03-29 06:42:21 +03:00
|
|
|
int fd = folly::openNoInt(infoPath.value().c_str(), O_RDONLY | O_CLOEXEC);
|
2017-02-11 01:16:00 +03:00
|
|
|
if (fd >= 0) {
|
|
|
|
// This is an existing overlay directory.
|
|
|
|
// Read the info file and make sure we are compatible with its version.
|
2018-03-29 06:42:21 +03:00
|
|
|
infoFile_ = File{fd, /* ownsFd */ true};
|
2017-07-01 05:00:03 +03:00
|
|
|
readExistingOverlay(infoFile_.fd());
|
2017-02-11 01:16:00 +03:00
|
|
|
} else if (errno != ENOENT) {
|
|
|
|
folly::throwSystemError(
|
|
|
|
"error reading eden overlay info file ", infoPath.stringPiece());
|
|
|
|
} else {
|
|
|
|
// This is a brand new overlay directory.
|
|
|
|
initNewOverlay();
|
2018-03-29 06:42:21 +03:00
|
|
|
infoFile_ = File{infoPath.value().c_str(), O_RDONLY | O_CLOEXEC};
|
2017-07-01 05:00:03 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
if (!infoFile_.try_lock()) {
|
|
|
|
folly::throwSystemError("failed to acquire overlay lock on ", infoPath);
|
2016-09-10 02:56:02 +03:00
|
|
|
}
|
2018-03-29 06:42:21 +03:00
|
|
|
|
|
|
|
// Open a handle on the overlay directory itself
|
|
|
|
int dirFd =
|
|
|
|
open(localDir_.c_str(), O_RDONLY | O_PATH | O_DIRECTORY | O_CLOEXEC);
|
|
|
|
folly::checkUnixError(
|
|
|
|
dirFd, "error opening overlay directory handle for ", localDir_.value());
|
|
|
|
dirFile_ = File{dirFd, /* ownsFd */ true};
|
2016-09-10 02:56:02 +03:00
|
|
|
}
|
|
|
|
|
2017-02-11 01:16:00 +03:00
|
|
|
bool Overlay::isOldFormatOverlay() const {
|
|
|
|
auto oldDir = localDir_ + PathComponentPiece{kOverlayTree};
|
|
|
|
struct stat s;
|
|
|
|
if (lstat(oldDir.value().c_str(), &s) == 0) {
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
2016-09-10 02:56:02 +03:00
|
|
|
|
2017-02-11 01:16:00 +03:00
|
|
|
void Overlay::readExistingOverlay(int infoFD) {
|
|
|
|
// Read the info file header
|
|
|
|
std::array<uint8_t, kInfoHeaderSize> infoHeader;
|
|
|
|
auto sizeRead = folly::readFull(infoFD, infoHeader.data(), infoHeader.size());
|
|
|
|
folly::checkUnixError(
|
|
|
|
sizeRead,
|
|
|
|
"error reading from overlay info file in ",
|
|
|
|
localDir_.stringPiece());
|
|
|
|
if (sizeRead != infoHeader.size()) {
|
|
|
|
throw std::runtime_error(folly::to<string>(
|
|
|
|
"truncated info file in overlay directory ", localDir_));
|
|
|
|
}
|
|
|
|
// Verify the magic value is correct
|
|
|
|
if (memcmp(
|
|
|
|
infoHeader.data(),
|
|
|
|
kInfoHeaderMagic.data(),
|
|
|
|
kInfoHeaderMagic.size()) != 0) {
|
|
|
|
throw std::runtime_error(
|
|
|
|
folly::to<string>("bad data in overlay info file for ", localDir_));
|
|
|
|
}
|
|
|
|
// Extract the version number
|
|
|
|
uint32_t version;
|
|
|
|
memcpy(
|
|
|
|
&version, infoHeader.data() + kInfoHeaderMagic.size(), sizeof(version));
|
|
|
|
version = folly::Endian::big(version);
|
|
|
|
|
|
|
|
// Make sure we understand this version number
|
|
|
|
if (version != kOverlayVersion) {
|
|
|
|
throw std::runtime_error(folly::to<string>(
|
|
|
|
"Unsupported eden overlay format ", version, " in ", localDir_));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void Overlay::initNewOverlay() {
|
|
|
|
// Make sure the overlay directory itself exists. It's fine if it already
|
|
|
|
// exists (although presumably it should be empty).
|
|
|
|
auto result = ::mkdir(localDir_.value().c_str(), 0755);
|
|
|
|
if (result != 0 && errno != EEXIST) {
|
|
|
|
folly::throwSystemError(
|
|
|
|
"error creating eden overlay directory ", localDir_.stringPiece());
|
|
|
|
}
|
|
|
|
auto localDirFile = File(localDir_.stringPiece(), O_RDONLY);
|
|
|
|
|
|
|
|
// We split the inode files across 256 subdirectories.
|
|
|
|
// Populate these subdirectories now.
|
|
|
|
std::array<char, 3> subdirPath;
|
|
|
|
subdirPath[2] = '\0';
|
2018-02-27 23:40:30 +03:00
|
|
|
for (uint64_t n = 0; n < 256; ++n) {
|
2017-02-11 01:16:00 +03:00
|
|
|
formatSubdirPath(MutableStringPiece{subdirPath.data(), 2}, n);
|
|
|
|
result = ::mkdirat(localDirFile.fd(), subdirPath.data(), 0755);
|
|
|
|
if (result != 0 && errno != EEXIST) {
|
|
|
|
folly::throwSystemError(
|
|
|
|
"error creating eden overlay directory ",
|
|
|
|
StringPiece{subdirPath.data()});
|
2016-09-10 02:56:02 +03:00
|
|
|
}
|
|
|
|
}
|
2017-02-11 01:16:00 +03:00
|
|
|
|
|
|
|
// For now we just write a simple header, with a magic number to identify
|
|
|
|
// this as an eden overlay file, and the version number of the overlay
|
|
|
|
// format.
|
|
|
|
std::array<uint8_t, kInfoHeaderSize> infoHeader;
|
|
|
|
memcpy(infoHeader.data(), kInfoHeaderMagic.data(), kInfoHeaderMagic.size());
|
|
|
|
auto version = folly::Endian::big(kOverlayVersion);
|
|
|
|
memcpy(
|
|
|
|
infoHeader.data() + kInfoHeaderMagic.size(), &version, sizeof(version));
|
|
|
|
|
|
|
|
auto infoPath = localDir_ + PathComponentPiece{kInfoFile};
|
|
|
|
folly::writeFileAtomic(
|
|
|
|
infoPath.stringPiece(), ByteRange(infoHeader.data(), infoHeader.size()));
|
|
|
|
}
|
|
|
|
|
2018-01-03 03:25:03 +03:00
|
|
|
Optional<TreeInode::Dir> Overlay::loadOverlayDir(
|
2018-03-20 03:01:15 +03:00
|
|
|
InodeNumber inodeNumber,
|
2018-03-29 06:42:21 +03:00
|
|
|
InodeMap* inodeMap) {
|
2017-08-11 21:34:52 +03:00
|
|
|
TreeInode::Dir result;
|
|
|
|
auto dirData = deserializeOverlayDir(inodeNumber, result.timeStamps);
|
2017-02-11 01:16:00 +03:00
|
|
|
if (!dirData.hasValue()) {
|
|
|
|
return folly::none;
|
|
|
|
}
|
|
|
|
const auto& dir = dirData.value();
|
2016-09-10 02:56:02 +03:00
|
|
|
|
2018-03-24 04:17:05 +03:00
|
|
|
bool shouldMigrateToNewFormat = false;
|
|
|
|
|
2016-09-10 02:56:02 +03:00
|
|
|
for (auto& iter : dir.entries) {
|
|
|
|
const auto& name = iter.first;
|
|
|
|
const auto& value = iter.second;
|
|
|
|
|
2018-03-24 04:17:05 +03:00
|
|
|
bool isMaterialized = !value.__isset.hash || value.hash.empty();
|
|
|
|
InodeNumber ino;
|
|
|
|
if (value.inodeNumber) {
|
|
|
|
ino = InodeNumber::fromThrift(value.inodeNumber);
|
|
|
|
} else {
|
|
|
|
ino = inodeMap->allocateInodeNumber();
|
|
|
|
shouldMigrateToNewFormat = true;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (isMaterialized) {
|
|
|
|
result.entries.emplace(PathComponentPiece{name}, value.mode, ino);
|
2017-02-11 01:16:00 +03:00
|
|
|
} else {
|
2018-03-24 04:17:05 +03:00
|
|
|
auto hash = Hash{folly::ByteRange{folly::StringPiece{value.hash}}};
|
|
|
|
result.entries.emplace(PathComponentPiece{name}, value.mode, ino, hash);
|
2016-09-10 02:56:02 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-03-24 04:17:05 +03:00
|
|
|
if (shouldMigrateToNewFormat) {
|
|
|
|
saveOverlayDir(inodeNumber, result);
|
|
|
|
}
|
|
|
|
|
2016-09-10 02:56:02 +03:00
|
|
|
return folly::Optional<TreeInode::Dir>(std::move(result));
|
|
|
|
}
|
|
|
|
|
2018-03-29 06:42:21 +03:00
|
|
|
void Overlay::saveOverlayDir(
|
|
|
|
InodeNumber inodeNumber,
|
|
|
|
const TreeInode::Dir& dir) {
|
2017-07-26 05:52:59 +03:00
|
|
|
// TODO: T20282158 clean up access of child inode information.
|
|
|
|
//
|
2016-09-10 02:56:02 +03:00
|
|
|
// Translate the data to the thrift equivalents
|
|
|
|
overlay::OverlayDir odir;
|
|
|
|
|
2018-02-15 04:40:39 +03:00
|
|
|
for (auto& entIter : dir.entries) {
|
2016-09-10 02:56:02 +03:00
|
|
|
const auto& entName = entIter.first;
|
2018-01-31 01:59:14 +03:00
|
|
|
const auto& ent = entIter.second;
|
2016-09-10 02:56:02 +03:00
|
|
|
|
|
|
|
overlay::OverlayEntry oent;
|
2018-01-31 01:59:14 +03:00
|
|
|
oent.mode = ent.getModeUnsafe();
|
2018-03-24 04:17:05 +03:00
|
|
|
oent.inodeNumber = ent.getInodeNumber().get();
|
|
|
|
bool isMaterialized = ent.isMaterialized();
|
|
|
|
if (!isMaterialized) {
|
2018-01-31 01:59:14 +03:00
|
|
|
auto entHash = ent.getHash();
|
2017-04-18 01:30:38 +03:00
|
|
|
auto bytes = entHash.getBytes();
|
2018-03-24 04:17:05 +03:00
|
|
|
oent.set_hash(std::string{reinterpret_cast<const char*>(bytes.data()),
|
|
|
|
bytes.size()});
|
2016-09-10 02:56:02 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
odir.entries.emplace(
|
|
|
|
std::make_pair(entName.stringPiece().str(), std::move(oent)));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Ask thrift to serialize it.
|
|
|
|
auto serializedData = CompactSerializer::serialize<std::string>(odir);
|
|
|
|
|
2018-03-29 06:42:21 +03:00
|
|
|
// Create the header
|
2018-02-08 02:01:01 +03:00
|
|
|
auto header =
|
2018-02-15 04:40:39 +03:00
|
|
|
createHeader(kHeaderIdentifierDir, kHeaderVersion, dir.timeStamps);
|
2017-07-04 10:18:17 +03:00
|
|
|
|
2018-03-29 06:42:21 +03:00
|
|
|
std::array<struct iovec, 2> iov;
|
|
|
|
iov[0].iov_base = header.data();
|
|
|
|
iov[0].iov_len = header.size();
|
|
|
|
iov[1].iov_base = const_cast<char*>(serializedData.data());
|
|
|
|
iov[1].iov_len = serializedData.size();
|
|
|
|
(void)createOverlayFileImpl(inodeNumber, iov.data(), iov.size());
|
2017-02-11 01:16:00 +03:00
|
|
|
}
|
2016-09-10 02:56:02 +03:00
|
|
|
|
2018-03-29 06:42:21 +03:00
|
|
|
void Overlay::removeOverlayData(InodeNumber inodeNumber) {
|
|
|
|
InodePath path;
|
|
|
|
getFilePath(inodeNumber, path);
|
|
|
|
if (::unlinkat(dirFile_.fd(), path.data(), 0) != 0 && errno != ENOENT) {
|
2017-02-11 01:16:00 +03:00
|
|
|
folly::throwSystemError("error unlinking overlay file: ", path);
|
2016-09-19 22:48:09 +03:00
|
|
|
}
|
2017-02-11 01:16:00 +03:00
|
|
|
}
|
2016-09-10 02:56:02 +03:00
|
|
|
|
2018-03-20 03:01:15 +03:00
|
|
|
InodeNumber Overlay::getMaxRecordedInode() {
|
2017-02-11 01:16:00 +03:00
|
|
|
// TODO: We should probably store the max inode number in the header file
|
|
|
|
// during graceful unmount. When opening an overlay we can then simply read
|
|
|
|
// back the max inode number from this file if the overlay was shut down
|
|
|
|
// cleanly last time.
|
|
|
|
//
|
|
|
|
// We would only then need to do a scan if the overlay was not cleanly shut
|
|
|
|
// down.
|
|
|
|
//
|
|
|
|
// For now we always do a scan.
|
2016-09-10 02:56:02 +03:00
|
|
|
|
2017-02-11 01:16:00 +03:00
|
|
|
// Walk the root directory downwards to find all (non-unlinked) directory
|
|
|
|
// inodes stored in the overlay.
|
|
|
|
//
|
|
|
|
// TODO: It would be nicer if each overlay file contained a short header so
|
|
|
|
// we could tell if it was a file or directory. This way we could do a
|
|
|
|
// simpler scan of opening every single file. For now we have to walk the
|
|
|
|
// directory tree from the root downwards.
|
2018-02-27 23:40:30 +03:00
|
|
|
auto maxInode = kRootNodeId;
|
2018-03-20 03:01:15 +03:00
|
|
|
std::vector<InodeNumber> toProcess;
|
2018-02-27 23:40:30 +03:00
|
|
|
toProcess.push_back(maxInode);
|
2017-02-11 01:16:00 +03:00
|
|
|
while (!toProcess.empty()) {
|
|
|
|
auto dirInodeNumber = toProcess.back();
|
|
|
|
toProcess.pop_back();
|
2016-09-10 02:56:02 +03:00
|
|
|
|
2018-02-01 23:21:03 +03:00
|
|
|
InodeTimestamps timeStamps;
|
2017-08-11 21:34:52 +03:00
|
|
|
auto dir = deserializeOverlayDir(dirInodeNumber, timeStamps);
|
2017-02-11 01:16:00 +03:00
|
|
|
if (!dir.hasValue()) {
|
|
|
|
continue;
|
|
|
|
}
|
2016-09-10 02:56:02 +03:00
|
|
|
|
2017-02-11 01:16:00 +03:00
|
|
|
for (const auto& entry : dir.value().entries) {
|
2018-02-27 23:40:30 +03:00
|
|
|
if (entry.second.inodeNumber == 0) {
|
2017-02-11 01:16:00 +03:00
|
|
|
continue;
|
|
|
|
}
|
2018-03-20 03:01:15 +03:00
|
|
|
auto entryInode = InodeNumber::fromThrift(entry.second.inodeNumber);
|
2017-02-11 01:16:00 +03:00
|
|
|
maxInode = std::max(maxInode, entryInode);
|
|
|
|
if (mode_to_dtype(entry.second.mode) == dtype_t::Dir) {
|
2018-02-27 23:40:30 +03:00
|
|
|
toProcess.push_back(entryInode);
|
2017-02-11 01:16:00 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2016-09-10 02:56:02 +03:00
|
|
|
|
2017-02-11 01:16:00 +03:00
|
|
|
// Look through the subdirectories and increment maxInode based on the
|
|
|
|
// filenames we see. This is needed in case there are unlinked inodes
|
|
|
|
// present.
|
|
|
|
std::array<char, 2> subdir;
|
2018-02-27 23:40:30 +03:00
|
|
|
for (uint64_t n = 0; n < 256; ++n) {
|
2017-02-11 01:16:00 +03:00
|
|
|
formatSubdirPath(MutableStringPiece{subdir.data(), subdir.size()}, n);
|
|
|
|
auto subdirPath = localDir_ +
|
|
|
|
PathComponentPiece{StringPiece{subdir.data(), subdir.size()}};
|
2016-09-10 02:56:02 +03:00
|
|
|
|
2017-02-11 01:16:00 +03:00
|
|
|
auto boostPath = boost::filesystem::path{subdirPath.value().c_str()};
|
|
|
|
for (const auto& entry : boost::filesystem::directory_iterator(boostPath)) {
|
|
|
|
auto entryInodeNumber =
|
2018-02-27 23:40:30 +03:00
|
|
|
folly::tryTo<uint64_t>(entry.path().filename().string());
|
2017-02-11 01:16:00 +03:00
|
|
|
if (entryInodeNumber.hasValue()) {
|
2018-03-20 03:01:15 +03:00
|
|
|
maxInode = std::max(maxInode, InodeNumber{entryInodeNumber.value()});
|
2017-02-11 01:16:00 +03:00
|
|
|
}
|
|
|
|
}
|
2016-09-19 22:48:09 +03:00
|
|
|
}
|
2017-02-11 01:16:00 +03:00
|
|
|
|
|
|
|
return maxInode;
|
2016-09-10 02:56:02 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
const AbsolutePath& Overlay::getLocalDir() const {
|
|
|
|
return localDir_;
|
|
|
|
}
|
|
|
|
|
2018-03-29 06:42:21 +03:00
|
|
|
size_t Overlay::getFilePath(InodeNumber inodeNumber, InodePath& outPath) {
|
|
|
|
formatSubdirPath(MutableStringPiece{outPath.data(), 2}, inodeNumber.get());
|
|
|
|
outPath[2] = '/';
|
|
|
|
auto index =
|
|
|
|
folly::uint64ToBufferUnsafe(inodeNumber.get(), outPath.data() + 3);
|
|
|
|
DCHECK_LT(index + 3, outPath.size());
|
|
|
|
outPath[index + 3] = '\0';
|
|
|
|
return index + 3;
|
2017-02-11 01:16:00 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
Optional<overlay::OverlayDir> Overlay::deserializeOverlayDir(
|
2018-03-20 03:01:15 +03:00
|
|
|
InodeNumber inodeNumber,
|
2018-02-01 23:21:03 +03:00
|
|
|
InodeTimestamps& timeStamps) const {
|
2018-03-29 06:42:21 +03:00
|
|
|
// Open the file. Return folly::none if the file does not exist.
|
|
|
|
InodePath path;
|
|
|
|
getFilePath(inodeNumber, path);
|
|
|
|
int fd = openat(dirFile_.fd(), path.data(), O_RDWR | O_CLOEXEC | O_NOFOLLOW);
|
|
|
|
if (fd == -1) {
|
|
|
|
int err = errno;
|
|
|
|
if (err == ENOENT) {
|
|
|
|
// There is no overlay here
|
|
|
|
return folly::none;
|
|
|
|
}
|
|
|
|
folly::throwSystemErrorExplicit(
|
|
|
|
err,
|
|
|
|
"error opening overlay file for inode ",
|
|
|
|
inodeNumber,
|
|
|
|
" in ",
|
|
|
|
localDir_);
|
|
|
|
}
|
|
|
|
folly::File file{fd, /* ownsFd */ true};
|
2017-02-11 01:16:00 +03:00
|
|
|
|
2018-03-29 06:42:21 +03:00
|
|
|
// Read the file data
|
2017-02-11 01:16:00 +03:00
|
|
|
std::string serializedData;
|
2018-03-29 06:42:21 +03:00
|
|
|
if (!folly::readFile(file.fd(), serializedData)) {
|
2017-02-11 01:16:00 +03:00
|
|
|
int err = errno;
|
|
|
|
if (err == ENOENT) {
|
|
|
|
// There is no overlay here
|
|
|
|
return folly::none;
|
|
|
|
}
|
2018-03-29 06:42:21 +03:00
|
|
|
folly::throwSystemErrorExplicit(errno, "failed to read ", path);
|
2017-02-11 01:16:00 +03:00
|
|
|
}
|
2017-07-04 10:18:17 +03:00
|
|
|
|
|
|
|
// Removing header and deserializing the contents
|
|
|
|
if (serializedData.size() < kHeaderLength) {
|
|
|
|
// Something Wrong with the file(may be corrupted)
|
|
|
|
folly::throwSystemErrorExplicit(
|
|
|
|
EIO,
|
|
|
|
"Overlay file ",
|
|
|
|
path,
|
|
|
|
" is too short for header: size=",
|
|
|
|
serializedData.size());
|
|
|
|
}
|
|
|
|
|
|
|
|
StringPiece header{serializedData, 0, kHeaderLength};
|
2017-08-05 06:14:18 +03:00
|
|
|
// validate header and get the timestamps
|
2017-08-11 21:34:52 +03:00
|
|
|
parseHeader(header, kHeaderIdentifierDir, timeStamps);
|
2017-08-05 06:14:18 +03:00
|
|
|
|
2017-07-04 10:18:17 +03:00
|
|
|
StringPiece contents{serializedData};
|
|
|
|
contents.advance(kHeaderLength);
|
|
|
|
|
|
|
|
return CompactSerializer::deserialize<overlay::OverlayDir>(contents);
|
|
|
|
}
|
|
|
|
|
2018-03-29 06:42:21 +03:00
|
|
|
std::array<uint8_t, Overlay::kHeaderLength> Overlay::createHeader(
|
|
|
|
folly::StringPiece identifier,
|
2017-07-04 10:18:17 +03:00
|
|
|
uint32_t version,
|
2018-02-08 02:01:01 +03:00
|
|
|
const InodeTimestamps& timestamps) {
|
2018-03-29 06:42:21 +03:00
|
|
|
std::array<uint8_t, kHeaderLength> headerStorage;
|
|
|
|
IOBuf header{IOBuf::WRAP_BUFFER, folly::MutableByteRange{headerStorage}};
|
|
|
|
header.clear();
|
2017-07-04 10:18:17 +03:00
|
|
|
folly::io::Appender appender(&header, 0);
|
2018-03-29 06:42:21 +03:00
|
|
|
|
2017-07-04 10:18:17 +03:00
|
|
|
appender.push(identifier);
|
|
|
|
appender.writeBE(version);
|
2018-02-16 03:00:44 +03:00
|
|
|
auto atime = timestamps.atime.toTimespec();
|
|
|
|
auto ctime = timestamps.ctime.toTimespec();
|
|
|
|
auto mtime = timestamps.mtime.toTimespec();
|
|
|
|
appender.writeBE<uint64_t>(atime.tv_sec);
|
|
|
|
appender.writeBE<uint64_t>(atime.tv_nsec);
|
|
|
|
appender.writeBE<uint64_t>(ctime.tv_sec);
|
|
|
|
appender.writeBE<uint64_t>(ctime.tv_nsec);
|
|
|
|
appender.writeBE<uint64_t>(mtime.tv_sec);
|
|
|
|
appender.writeBE<uint64_t>(mtime.tv_nsec);
|
2017-07-04 10:18:17 +03:00
|
|
|
auto paddingSize = kHeaderLength - header.length();
|
|
|
|
appender.ensure(paddingSize);
|
|
|
|
memset(appender.writableData(), 0, paddingSize);
|
|
|
|
appender.append(paddingSize);
|
|
|
|
|
2018-03-29 06:42:21 +03:00
|
|
|
return headerStorage;
|
2016-09-10 02:56:02 +03:00
|
|
|
}
|
2017-07-14 03:11:36 +03:00
|
|
|
|
|
|
|
// Helper function to open,validate,
|
|
|
|
// get file pointer of an overlay file
|
2017-08-05 06:14:18 +03:00
|
|
|
folly::File Overlay::openFile(
|
2018-03-28 22:54:24 +03:00
|
|
|
InodeNumber inodeNumber,
|
2017-08-05 06:14:18 +03:00
|
|
|
folly::StringPiece headerId,
|
2018-02-01 23:21:03 +03:00
|
|
|
InodeTimestamps& timeStamps) {
|
2017-07-14 03:11:36 +03:00
|
|
|
// Open the overlay file
|
2018-03-29 06:42:21 +03:00
|
|
|
auto file = openFileNoVerify(inodeNumber);
|
2017-07-14 03:11:36 +03:00
|
|
|
|
|
|
|
// Read the contents
|
|
|
|
std::string contents;
|
2018-03-29 06:42:21 +03:00
|
|
|
if (!folly::readFile(file.fd(), contents, kHeaderLength)) {
|
|
|
|
folly::throwSystemErrorExplicit(
|
|
|
|
errno,
|
|
|
|
"failed to read overlay file for inode ",
|
|
|
|
inodeNumber,
|
|
|
|
" in ",
|
|
|
|
localDir_);
|
|
|
|
}
|
2017-07-14 03:11:36 +03:00
|
|
|
|
2017-08-05 06:14:18 +03:00
|
|
|
StringPiece header{contents};
|
2017-08-11 21:34:52 +03:00
|
|
|
parseHeader(header, headerId, timeStamps);
|
2017-07-14 03:11:36 +03:00
|
|
|
return file;
|
|
|
|
}
|
|
|
|
|
2018-03-28 22:54:24 +03:00
|
|
|
folly::File Overlay::openFileNoVerify(InodeNumber inodeNumber) {
|
2018-03-29 06:42:21 +03:00
|
|
|
InodePath path;
|
|
|
|
getFilePath(inodeNumber, path);
|
|
|
|
|
|
|
|
int fd = openat(dirFile_.fd(), path.data(), O_RDWR | O_CLOEXEC | O_NOFOLLOW);
|
|
|
|
folly::checkUnixError(
|
|
|
|
fd,
|
|
|
|
"error opening overlay file for inode ",
|
|
|
|
inodeNumber,
|
|
|
|
" in ",
|
|
|
|
localDir_);
|
|
|
|
return folly::File{fd, /* ownsFd */ true};
|
2018-03-28 22:54:24 +03:00
|
|
|
}
|
|
|
|
|
2018-03-29 06:42:21 +03:00
|
|
|
folly::File Overlay::createOverlayFileImpl(
|
|
|
|
InodeNumber inodeNumber,
|
|
|
|
iovec* iov,
|
|
|
|
size_t iovCount) {
|
|
|
|
// We do not use mkstemp() to create the temporary file, since there is no
|
|
|
|
// mkstempat() equivalent that can create files relative to dirFile_. We
|
|
|
|
// simply create the file with a fixed suffix, and do not use O_EXCL. This
|
|
|
|
// is not a security risk since only the current user should have permission
|
|
|
|
// to create files inside the overlay directory, so no one else can create
|
|
|
|
// symlinks inside the overlay directory. We also open the temporary file
|
|
|
|
// using O_NOFOLLOW.
|
|
|
|
//
|
|
|
|
// We could potentially use O_TMPFILE followed by linkat() to commit the
|
|
|
|
// file. However this may not be supported by all filesystems, and seems to
|
|
|
|
// provide minimal benefits for our use case.
|
|
|
|
constexpr auto tmpSuffix = ".tmp\0"_sp;
|
|
|
|
InodePath path;
|
|
|
|
std::array<char, kMaxPathLength + tmpSuffix.size()> tmpPath;
|
|
|
|
auto pathLength = getFilePath(inodeNumber, path);
|
|
|
|
memcpy(tmpPath.data(), path.data(), pathLength);
|
|
|
|
memcpy(tmpPath.data() + pathLength, tmpSuffix.data(), tmpSuffix.size());
|
|
|
|
|
|
|
|
auto tmpFD = openat(
|
|
|
|
dirFile_.fd(),
|
|
|
|
tmpPath.data(),
|
|
|
|
O_CREAT | O_RDWR | O_CLOEXEC | O_NOFOLLOW,
|
|
|
|
0600);
|
|
|
|
folly::checkUnixError(
|
|
|
|
tmpFD,
|
|
|
|
"failed to create temporary overlay file for inode ",
|
|
|
|
inodeNumber,
|
|
|
|
" in ",
|
|
|
|
localDir_);
|
|
|
|
folly::File file{tmpFD, /* ownsFd */ true};
|
|
|
|
bool success = false;
|
|
|
|
SCOPE_EXIT {
|
|
|
|
if (!success) {
|
|
|
|
unlinkat(dirFile_.fd(), tmpPath.data(), 0);
|
|
|
|
}
|
|
|
|
};
|
2017-07-14 03:11:36 +03:00
|
|
|
|
2018-03-29 06:42:21 +03:00
|
|
|
auto sizeWritten = folly::writevFull(tmpFD, iov, iovCount);
|
|
|
|
folly::checkUnixError(
|
|
|
|
sizeWritten,
|
|
|
|
"error writing to overlay file for inode ",
|
|
|
|
inodeNumber,
|
|
|
|
" in ",
|
|
|
|
localDir_);
|
|
|
|
|
|
|
|
auto returnCode = folly::fdatasyncNoInt(tmpFD);
|
|
|
|
folly::checkUnixError(
|
|
|
|
returnCode,
|
|
|
|
"error flushing data to overlay file for inode ",
|
|
|
|
inodeNumber,
|
|
|
|
" in ",
|
|
|
|
localDir_);
|
|
|
|
|
|
|
|
returnCode =
|
|
|
|
renameat(dirFile_.fd(), tmpPath.data(), dirFile_.fd(), path.data());
|
|
|
|
folly::checkUnixError(
|
|
|
|
returnCode,
|
|
|
|
"error committing overlay file for inode ",
|
|
|
|
inodeNumber,
|
|
|
|
" in ",
|
|
|
|
localDir_);
|
|
|
|
// We do not want to unlink the temporary file on exit now that we have
|
|
|
|
// successfully renamed it.
|
|
|
|
success = true;
|
|
|
|
|
|
|
|
return file;
|
2017-07-14 03:11:36 +03:00
|
|
|
}
|
|
|
|
|
2018-01-03 03:25:03 +03:00
|
|
|
folly::File Overlay::createOverlayFile(
|
2018-03-28 22:54:24 +03:00
|
|
|
InodeNumber inodeNumber,
|
2018-03-28 22:54:25 +03:00
|
|
|
const InodeTimestamps& timestamps,
|
|
|
|
ByteRange contents) {
|
2018-03-29 06:42:21 +03:00
|
|
|
auto header = createHeader(kHeaderIdentifierFile, kHeaderVersion, timestamps);
|
2017-07-14 03:11:36 +03:00
|
|
|
|
2018-03-29 06:42:21 +03:00
|
|
|
std::array<struct iovec, 2> iov;
|
|
|
|
iov[0].iov_base = header.data();
|
|
|
|
iov[0].iov_len = header.size();
|
|
|
|
iov[1].iov_base = const_cast<uint8_t*>(contents.data());
|
|
|
|
iov[1].iov_len = contents.size();
|
|
|
|
return createOverlayFileImpl(inodeNumber, iov.data(), iov.size());
|
2017-07-14 03:11:36 +03:00
|
|
|
}
|
2017-08-05 06:14:18 +03:00
|
|
|
|
2018-03-28 22:54:24 +03:00
|
|
|
folly::File Overlay::createOverlayFile(
|
|
|
|
InodeNumber inodeNumber,
|
|
|
|
const InodeTimestamps& timestamps,
|
|
|
|
const IOBuf& contents) {
|
2018-03-29 06:42:21 +03:00
|
|
|
// In the common case where there is just one element in the chain, use the
|
|
|
|
// ByteRange version of createOverlayFile() to avoid having to allocate the
|
|
|
|
// iovec array on the heap.
|
|
|
|
if (contents.next() == &contents) {
|
|
|
|
return createOverlayFile(
|
|
|
|
inodeNumber, timestamps, ByteRange{contents.data(), contents.length()});
|
|
|
|
}
|
2018-03-28 22:54:24 +03:00
|
|
|
|
2018-03-29 06:42:21 +03:00
|
|
|
auto header = createHeader(kHeaderIdentifierFile, kHeaderVersion, timestamps);
|
2018-03-28 22:54:24 +03:00
|
|
|
|
2018-03-29 06:42:21 +03:00
|
|
|
fbvector<struct iovec> iov;
|
|
|
|
iov.resize(1);
|
|
|
|
iov[0].iov_base = header.data();
|
|
|
|
iov[0].iov_len = header.size();
|
|
|
|
contents.appendToIov(&iov);
|
2018-03-28 22:54:24 +03:00
|
|
|
|
2018-03-29 06:42:21 +03:00
|
|
|
return createOverlayFileImpl(inodeNumber, iov.data(), iov.size());
|
2018-03-28 22:54:24 +03:00
|
|
|
}
|
|
|
|
|
2017-08-05 06:14:18 +03:00
|
|
|
void Overlay::parseHeader(
|
|
|
|
folly::StringPiece header,
|
|
|
|
folly::StringPiece headerId,
|
2018-02-16 03:00:44 +03:00
|
|
|
InodeTimestamps& timestamps) {
|
2018-03-28 22:54:24 +03:00
|
|
|
IOBuf buf(IOBuf::WRAP_BUFFER, ByteRange{header});
|
2017-08-05 06:14:18 +03:00
|
|
|
folly::io::Cursor cursor(&buf);
|
|
|
|
|
|
|
|
// Validate header identifier
|
|
|
|
auto id = cursor.readFixedString(kHeaderIdentifierDir.size());
|
|
|
|
StringPiece identifier{id};
|
|
|
|
if (identifier.compare(headerId) != 0) {
|
|
|
|
folly::throwSystemError(
|
|
|
|
EIO,
|
|
|
|
"unexpected overlay header identifier : ",
|
|
|
|
folly::hexlify(ByteRange{identifier}));
|
|
|
|
}
|
|
|
|
|
|
|
|
// Validate header version
|
|
|
|
auto version = cursor.readBE<uint32_t>();
|
|
|
|
if (version != kHeaderVersion) {
|
|
|
|
folly::throwSystemError(EIO, "Unexpected overlay version :", version);
|
|
|
|
}
|
2018-02-16 03:00:44 +03:00
|
|
|
timespec atime, ctime, mtime;
|
|
|
|
atime.tv_sec = cursor.readBE<uint64_t>();
|
|
|
|
atime.tv_nsec = cursor.readBE<uint64_t>();
|
|
|
|
ctime.tv_sec = cursor.readBE<uint64_t>();
|
|
|
|
ctime.tv_nsec = cursor.readBE<uint64_t>();
|
|
|
|
mtime.tv_sec = cursor.readBE<uint64_t>();
|
|
|
|
mtime.tv_nsec = cursor.readBE<uint64_t>();
|
|
|
|
timestamps.atime = atime;
|
|
|
|
timestamps.ctime = ctime;
|
|
|
|
timestamps.mtime = mtime;
|
2017-08-05 06:14:18 +03:00
|
|
|
}
|
|
|
|
// Helper function to update timestamps into overlay file
|
2017-08-11 21:34:52 +03:00
|
|
|
void Overlay::updateTimestampToHeader(
|
|
|
|
int fd,
|
2018-02-16 03:00:44 +03:00
|
|
|
const InodeTimestamps& timestamps) {
|
2017-08-05 06:14:18 +03:00
|
|
|
// Create a string piece with timestamps
|
|
|
|
std::array<uint64_t, 6> buf;
|
2018-03-28 22:54:24 +03:00
|
|
|
IOBuf iobuf(IOBuf::WRAP_BUFFER, buf.data(), sizeof(buf));
|
2018-02-16 03:00:44 +03:00
|
|
|
iobuf.clear();
|
|
|
|
|
|
|
|
folly::io::Appender appender(&iobuf, 0);
|
|
|
|
auto atime = timestamps.atime.toTimespec();
|
|
|
|
auto ctime = timestamps.ctime.toTimespec();
|
|
|
|
auto mtime = timestamps.mtime.toTimespec();
|
|
|
|
appender.writeBE<uint64_t>(atime.tv_sec);
|
|
|
|
appender.writeBE<uint64_t>(atime.tv_nsec);
|
|
|
|
appender.writeBE<uint64_t>(ctime.tv_sec);
|
|
|
|
appender.writeBE<uint64_t>(ctime.tv_nsec);
|
|
|
|
appender.writeBE<uint64_t>(mtime.tv_sec);
|
|
|
|
appender.writeBE<uint64_t>(mtime.tv_nsec);
|
2017-08-05 06:14:18 +03:00
|
|
|
|
|
|
|
// replace the timestamps of current header with the new timestamps
|
2018-02-16 03:00:44 +03:00
|
|
|
auto newHeader = iobuf.coalesce();
|
2017-08-05 06:14:18 +03:00
|
|
|
auto wrote = folly::pwriteNoInt(
|
|
|
|
fd,
|
|
|
|
newHeader.data(),
|
|
|
|
newHeader.size(),
|
|
|
|
kHeaderIdentifierDir.size() + sizeof(kHeaderVersion));
|
|
|
|
if (wrote == -1) {
|
|
|
|
folly::throwSystemError("pwriteNoInt failed");
|
|
|
|
}
|
|
|
|
if (wrote != newHeader.size()) {
|
|
|
|
folly::throwSystemError(
|
|
|
|
"writeNoInt wrote only ", wrote, " of ", newHeader.size(), " bytes");
|
|
|
|
}
|
|
|
|
}
|
2017-11-04 01:58:04 +03:00
|
|
|
} // namespace eden
|
|
|
|
} // namespace facebook
|