teach Takeover Server/Client how to pass NFS info

Summary:
Now that we have NFS mounts we need to support takeover for those mounts.
This commit teaches the takeover server and client to serialize and deserialize
NFS information.

NFS information includes:
- the mountd socket. Mountd is used to register mount points, and should keep
the same socket across graceful restarts.
- a connected socket for each of the NFS mount points.
- All the rest of the mount point information that we send for fuse mounts (with
the exception of the fuse negotiated parameters.

Future commits will teach the NFS components to graceful stop and yield this info
as well as restart with this info.

Reviewed By: xavierd

Differential Revision: D31802787

fbshipit-source-id: 667625589b77eed79f7e17390013a818d4be7068
This commit is contained in:
Katie Mancini 2021-11-17 17:09:58 -08:00 committed by Facebook GitHub Bot
parent 7bcd208273
commit 43fc599083
13 changed files with 552 additions and 133 deletions

View File

@ -1840,7 +1840,7 @@ void EdenMount::preparePostChannelCompletion(
getPath(),
checkoutConfig_->getClientDirectory(),
bindMounts,
folly::File{},
ProjFsChannelData{}, // placeholder
SerializedInodeMap{} // placeholder
));
#else
@ -1861,8 +1861,9 @@ void EdenMount::preparePostChannelCompletion(
getPath(),
checkoutConfig_->getClientDirectory(),
bindMounts,
std::move(variant.fuseDevice),
variant.fuseSettings,
FuseChannelData{
std::move(variant.fuseDevice),
variant.fuseSettings},
SerializedInodeMap{} // placeholder
));
} else {
@ -1874,10 +1875,8 @@ void EdenMount::preparePostChannelCompletion(
getPath(),
checkoutConfig_->getClientDirectory(),
bindMounts,
// TODO(xavierd): the next 2 fields should be a variant
// too.
folly::File(),
fuse_init_out{},
FuseChannelData{
folly::File(), fuse_init_out{}}, // TODO: NFS data
SerializedInodeMap{} // placeholder
));
}
@ -1918,8 +1917,8 @@ void EdenMount::channelInitSuccessful(
#endif
}
#ifndef _WIN32
void EdenMount::takeoverFuse(FuseChannelData takeoverData) {
#ifndef _WIN32
transitionState(State::INITIALIZED, State::STARTING);
try {
@ -1939,8 +1938,12 @@ void EdenMount::takeoverFuse(FuseChannelData takeoverData) {
transitionToFuseInitializationErrorState();
throw;
}
#else
throw std::runtime_error("Fuse not supported on this platform.");
#endif
}
#ifndef _WIN32
InodeMetadata EdenMount::getInitialInodeMetadata(mode_t mode) const {
auto owner = getOwner();
return InodeMetadata{

View File

@ -662,7 +662,6 @@ class EdenMount : public std::enable_shared_from_this<EdenMount> {
*/
FOLLY_NODISCARD folly::Future<folly::Unit> startChannel(bool readOnly);
#ifndef _WIN32
/**
* Take over a FUSE channel for an existing mount point.
*
@ -671,9 +670,10 @@ class EdenMount : public std::enable_shared_from_this<EdenMount> {
*
* If unmount() is called before takeoverFuse() is called, then takeoverFuse()
* throws an EdenMountCancelled exception.
*
* throws a runtime_error if fuse is not supported on this platform.
*/
void takeoverFuse(FuseChannelData takeoverData);
#endif
/**
* Obtains a future that will complete once the channel has wound down.

View File

@ -1263,7 +1263,8 @@ TEST(EdenMountState, newMountIsRunningAndOldMountIsShutDownAfterFuseTakeover) {
oldMount.getChannelCompletionFuture().within(kTimeout).getVia(
oldTestMount.getServerExecutor().get());
oldMount.shutdown(/*doTakeover=*/true).get(kTimeout);
newMount.takeoverFuse(FuseChannelData{std::move(takeoverData.fuseFD), {}});
auto& fuseChannelData = std::get<FuseChannelData>(takeoverData.channelInfo);
newMount.takeoverFuse(std::move(fuseChannelData));
EXPECT_EQ(oldMount.getState(), EdenMount::State::SHUT_DOWN);
EXPECT_EQ(newMount.getState(), EdenMount::State::RUNNING);

View File

@ -86,11 +86,12 @@ TEST(FuseTest, initMount) {
testMount.drainServerExecutor();
} while (!fuseCompletionFuture.isReady());
auto mountInfo = std::move(fuseCompletionFuture.value());
auto mountInfo = std::get<FuseChannelData>(
std::move(fuseCompletionFuture.value()).channelInfo);
// Since we closed the FUSE device from the "kernel" side the returned
// MountInfo should not contain a valid FUSE device any more.
EXPECT_FALSE(mountInfo.fuseFD);
EXPECT_FALSE(mountInfo.fd);
}
// Test destroying the EdenMount object while FUSE initialization is still
@ -180,11 +181,12 @@ TEST(FuseTest, destroyWithInitRace) {
if (initFuseSuccessful) {
// The FUSE completion future should also be signalled when the FuseChannel
// is destroyed.
auto mountInfo = std::move(completionFuture).get(250ms);
auto mountInfo = std::get<FuseChannelData>(
std::move(completionFuture).get(250ms).channelInfo);
// Since we just destroyed the EdenMount and the kernel-side of the FUSE
// channel was not stopped the returned MountInfo should contain the FUSE
// device.
EXPECT_TRUE(mountInfo.fuseFD);
EXPECT_TRUE(mountInfo.fd);
}
}

View File

@ -71,6 +71,7 @@
#include "eden/fs/utils/EdenError.h"
#include "eden/fs/utils/EnumValue.h"
#include "eden/fs/utils/FileUtils.h"
#include "eden/fs/utils/FsChannelTypes.h"
#include "eden/fs/utils/NfsSocket.h"
#include "eden/fs/utils/NotImplemented.h"
#include "eden/fs/utils/PathFuncs.h"
@ -572,9 +573,12 @@ Future<TakeoverData> EdenServer::stopMountsForTakeover(
[self = this,
edenMount = info.edenMount](TakeoverData::MountInfo takeover)
-> Future<optional<TakeoverData::MountInfo>> {
if (!takeover.fuseFD) {
auto fuseChannelInfo =
std::get_if<FuseChannelData>(&takeover.channelInfo);
if (!fuseChannelInfo || !fuseChannelInfo->fd) {
return std::nullopt;
}
// TODO: takeover for NFS
return self->serverState_->getPrivHelper()
->takeoverShutdown(edenMount->getPath().stringPiece())
.thenValue([takeover = std::move(takeover)](auto&&) mutable {
@ -1389,17 +1393,14 @@ Future<Unit> EdenServer::performTakeoverStart(
Future<Unit> EdenServer::completeTakeoverStart(
FOLLY_MAYBE_UNUSED std::shared_ptr<EdenMount> edenMount,
FOLLY_MAYBE_UNUSED TakeoverData::MountInfo&& info) {
#ifndef _WIN32
FuseChannelData channelData;
channelData.fd = std::move(info.fuseFD);
channelData.connInfo = info.connInfo;
// Start up the fuse workers.
return folly::makeFutureWith(
[&] { edenMount->takeoverFuse(std::move(channelData)); });
#else
NOT_IMPLEMENTED();
#endif // !_WIN32
if (auto channelData = std::get_if<FuseChannelData>(&info.channelInfo)) {
// Start up the fuse workers.
return folly::makeFutureWith(
[&] { edenMount->takeoverFuse(std::move(*channelData)); });
} else { // TODO: takeover for NFS
return folly::makeFuture<Unit>(std::runtime_error(fmt::format(
"Unsupported ChannelInfo Type: {}", info.channelInfo.index())));
}
}
folly::Future<std::shared_ptr<EdenMount>> EdenServer::mount(

View File

@ -9,14 +9,19 @@
#include "eden/fs/takeover/TakeoverData.h"
#include <memory>
#include <stdexcept>
#include <variant>
#include <folly/Format.h>
#include <folly/io/Cursor.h>
#include <folly/io/IOBuf.h>
#include <folly/logging/xlog.h>
#include <thrift/lib/cpp2/protocol/Serializer.h>
#include "eden/fs/utils/Bug.h"
#include "eden/fs/utils/UnixSocket.h"
using apache::thrift::CompactSerializer;
using folly::IOBuf;
@ -25,6 +30,26 @@ using std::string;
namespace facebook {
namespace eden {
namespace {
/**
* Determines the mount protocol for the mount point encoded in the mountInfo.
*/
TakeoverMountProtocol getMountProtocol(
const TakeoverData::MountInfo& mountInfo) {
if (std::holds_alternative<FuseChannelData>(mountInfo.channelInfo)) {
return TakeoverMountProtocol::FUSE;
} else if (std::holds_alternative<NfsChannelData>(mountInfo.channelInfo)) {
return TakeoverMountProtocol::NFS;
}
throw std::runtime_error(fmt::format(
"unrecognized mount protocol {} for mount: {}",
mountInfo.channelInfo.index(),
mountInfo.mountPath));
}
} // namespace
const std::set<int32_t> kSupportedTakeoverVersions{
TakeoverData::kTakeoverProtocolVersionOne,
TakeoverData::kTakeoverProtocolVersionThree,
@ -40,7 +65,8 @@ std::optional<int32_t> TakeoverData::computeCompatibleVersion(
// No better than the current best
continue;
}
if (supported.find(version) == supported.end()) {
if (std::find(supported.begin(), supported.end(), version) ==
supported.end()) {
// Not supported
continue;
}
@ -63,6 +89,11 @@ uint64_t TakeoverData::versionToCapabilites(int32_t version) {
return TakeoverCapabilities::FUSE |
TakeoverCapabilities::THRIFT_SERIALIZATION |
TakeoverCapabilities::PING;
case kTakeoverProtocolVersionFive:
return TakeoverCapabilities::FUSE | TakeoverCapabilities::MOUNT_TYPES |
TakeoverCapabilities::PING |
TakeoverCapabilities::THRIFT_SERIALIZATION |
TakeoverCapabilities::NFS;
}
throw std::runtime_error(fmt::format("Unsupported version: {}", version));
}
@ -87,18 +118,42 @@ int32_t TakeoverData::capabilitesToVersion(uint64_t capabilities) {
return kTakeoverProtocolVersionFour;
}
if (capabilities ==
(TakeoverCapabilities::FUSE | TakeoverCapabilities::MOUNT_TYPES |
TakeoverCapabilities::PING | TakeoverCapabilities::THRIFT_SERIALIZATION |
TakeoverCapabilities::NFS)) {
return kTakeoverProtocolVersionFive;
}
throw std::runtime_error(
fmt::format("Unsupported combination of capabilities: {}", capabilities));
}
bool TakeoverData::shouldSerdeNFSInfo(uint32_t protocolCapabilities) {
// 4 and below know nothing of NFS mounts. we introduce NFS support in version
// 5 and expect to continue to support NFS mounts beyond version 5.
return protocolCapabilities & TakeoverCapabilities::NFS;
}
void TakeoverData::serialize(
uint64_t protocolCapabilities,
UnixSocket::Message& msg) {
msg.data = serialize(protocolCapabilities);
msg.files.push_back(std::move(lockFile));
msg.files.push_back(std::move(thriftSocket));
if (shouldSerdeNFSInfo(protocolCapabilities)) {
XLOG(DBG7) << "serializing mountd socket: " << mountdServerSocket.fd();
msg.files.push_back(std::move(mountdServerSocket));
}
for (auto& mount : mountPoints) {
msg.files.push_back(std::move(mount.fuseFD));
if (auto fuseData = std::get_if<FuseChannelData>(&mount.channelInfo)) {
msg.files.push_back(std::move(fuseData->fd));
} else if (auto nfsData = std::get_if<NfsChannelData>(&mount.channelInfo)) {
msg.files.push_back(std::move(nfsData->nfsdSocketFd));
} else {
throw std::runtime_error("Unexpected Channel Type");
}
}
}
@ -165,7 +220,9 @@ TakeoverData TakeoverData::deserialize(UnixSocket::Message& msg) {
auto capabilities = TakeoverData::versionToCapabilites(protocolVersion);
auto data = TakeoverData::deserialize(capabilities, &msg.data);
constexpr auto mountPointFilesOffset = 2;
// when we serialize the mountd socket we have three general files instead
// of two
const auto mountPointFilesOffset = shouldSerdeNFSInfo(capabilities) ? 3 : 2;
// Add 2 here for the lock file and the thrift socket
if (data.mountPoints.size() + mountPointFilesOffset != msg.files.size()) {
@ -178,9 +235,20 @@ TakeoverData TakeoverData::deserialize(UnixSocket::Message& msg) {
}
data.lockFile = std::move(msg.files[0]);
data.thriftSocket = std::move(msg.files[1]);
if (shouldSerdeNFSInfo(capabilities)) {
data.mountdServerSocket = std::move(msg.files[2]);
XLOG(DBG1) << "Deserialized mountd Socket " << data.mountdServerSocket.fd();
}
for (size_t n = 0; n < data.mountPoints.size(); ++n) {
auto& mountInfo = data.mountPoints[n];
mountInfo.fuseFD = std::move(msg.files[n + mountPointFilesOffset]);
if (auto fuseData = std::get_if<FuseChannelData>(&mountInfo.channelInfo)) {
fuseData->fd = std::move(msg.files[n + mountPointFilesOffset]);
} else if (
auto nfsData = std::get_if<NfsChannelData>(&mountInfo.channelInfo)) {
nfsData->nfsdSocketFd = std::move(msg.files[n + mountPointFilesOffset]);
} else {
throw std::runtime_error("Unexpected Channel Type");
}
}
return data;
}
@ -198,6 +266,7 @@ int32_t TakeoverData::getProtocolVersion(IOBuf* buf) {
return kTakeoverProtocolVersionOne;
case kTakeoverProtocolVersionThree:
case kTakeoverProtocolVersionFour:
case kTakeoverProtocolVersionFive:
// Version 3 (there was no 2 because of how Version 1 used word values
// 1 and 2) doesn't care about this version byte, so we skip past it
// and let the underlying code decode the data
@ -220,7 +289,7 @@ TakeoverData TakeoverData::deserialize(
return deserializeCustom(buf);
}
if (serializationMethod == TakeoverCapabilities::THRIFT_SERIALIZATION) {
return deserializeThrift(buf);
return deserializeThrift(protocolCapabilities, buf);
}
throw std::runtime_error(fmt::format(
@ -262,37 +331,48 @@ IOBuf TakeoverData::serializeCustom() {
// Serialize each mount point
for (const auto& mount : mountPoints) {
// The mount path
const auto& pathStr = mount.mountPath.stringPiece();
app.writeBE<uint32_t>(pathStr.size());
app(pathStr);
auto mountProtocol = getMountProtocol(mount);
if (mountProtocol == TakeoverMountProtocol::FUSE) {
auto& channelData = std::get<FuseChannelData>(mount.channelInfo);
// The client configuration dir
const auto& clientStr = mount.stateDirectory.stringPiece();
app.writeBE<uint32_t>(clientStr.size());
app(clientStr);
// The mount path
const auto& pathStr = mount.mountPath.stringPiece();
app.writeBE<uint32_t>(pathStr.size());
app(pathStr);
// Number of bind mounts, followed by the bind mount paths
app.writeBE<uint32_t>(mount.bindMounts.size());
for (const auto& bindMount : mount.bindMounts) {
app.writeBE<uint32_t>(bindMount.stringPiece().size());
app(bindMount.stringPiece());
// The client configuration dir
const auto& clientStr = mount.stateDirectory.stringPiece();
app.writeBE<uint32_t>(clientStr.size());
app(clientStr);
// Number of bind mounts, followed by the bind mount paths
app.writeBE<uint32_t>(mount.bindMounts.size());
for (const auto& bindMount : mount.bindMounts) {
app.writeBE<uint32_t>(bindMount.stringPiece().size());
app(bindMount.stringPiece());
}
// Stuffing the fuse connection information in as a binary
// blob because we know that the endianness of the target
// machine must match the current system for a graceful
// takeover.
app.push(folly::StringPiece{
reinterpret_cast<const char*>(&channelData.connInfo),
sizeof(channelData.connInfo)});
// SerializedFileHandleMap has been removed so its size is always 0.
app.writeBE<uint32_t>(0);
auto serializedInodeMap =
CompactSerializer::serialize<std::string>(mount.inodeMap);
app.writeBE<uint32_t>(serializedInodeMap.size());
app.push(folly::StringPiece{serializedInodeMap});
} else {
throw std::runtime_error(fmt::format(
"version 1 of the protocol does not support serializing non-FUSE"
"mounts. problem mount: {} . protocol: {}",
mount.mountPath,
mountProtocol));
}
// Stuffing the fuse connection information in as a binary
// blob because we know that the endianness of the target
// machine must match the current system for a graceful
// takeover.
app.push(folly::StringPiece{
reinterpret_cast<const char*>(&mount.connInfo),
sizeof(mount.connInfo)});
// SerializedFileHandleMap has been removed so its size is always 0.
app.writeBE<uint32_t>(0);
auto serializedInodeMap =
CompactSerializer::serialize<std::string>(mount.inodeMap);
app.writeBE<uint32_t>(serializedInodeMap.size());
app.push(folly::StringPiece{serializedInodeMap});
}
return buf;
@ -381,14 +461,42 @@ TakeoverData TakeoverData::deserializeCustom(IOBuf* buf) {
AbsolutePath{mountPath},
AbsolutePath{stateDirectory},
std::move(bindMounts),
folly::File{},
connInfo,
FuseChannelData{folly::File{}, connInfo},
std::move(inodeMap));
}
return data;
}
bool canSerDeMountType(
uint64_t protocolCapabilities,
TakeoverMountProtocol mountProtocol) {
switch (mountProtocol) {
case TakeoverMountProtocol::FUSE:
return protocolCapabilities & TakeoverCapabilities::FUSE;
case TakeoverMountProtocol::NFS:
return protocolCapabilities & TakeoverCapabilities::NFS;
case TakeoverMountProtocol::UNKNOWN:
return false;
}
return false;
}
void checkCanSerDeMountType(
uint64_t protocolCapabilities,
TakeoverMountProtocol mountProtocol,
folly::StringPiece mountPath) {
if (!canSerDeMountType(protocolCapabilities, mountProtocol)) {
throw std::runtime_error(fmt::format(
"protocol does not support serializing/deserializing this type of "
"mounts. protocol capabilities: {}. problem mount: {}. mount protocol:"
" {}",
protocolCapabilities,
mountPath,
mountProtocol));
}
}
IOBuf TakeoverData::serializeThrift(uint64_t protocolCapabilities) {
SerializedTakeoverData serialized;
@ -411,6 +519,11 @@ IOBuf TakeoverData::serializeThrift(uint64_t protocolCapabilities) {
std::vector<SerializedMountInfo> serializedMounts;
for (const auto& mount : mountPoints) {
auto mountProtocol = getMountProtocol(mount);
checkCanSerDeMountType(
protocolCapabilities, mountProtocol, mount.mountPath.stringPiece());
SerializedMountInfo serializedMount;
*serializedMount.mountPath_ref() = mount.mountPath.stringPiece().str();
@ -422,16 +535,22 @@ IOBuf TakeoverData::serializeThrift(uint64_t protocolCapabilities) {
bindMount.stringPiece().str());
}
// Stuffing the fuse connection information in as a binary
// blob because we know that the endianness of the target
// machine must match the current system for a graceful
// takeover, and it saves us from re-encoding an operating
// system specific struct into a thrift file.
*serializedMount.connInfo_ref() = std::string{
reinterpret_cast<const char*>(&mount.connInfo), sizeof(mount.connInfo)};
if (auto fuseChannelInfo =
std::get_if<FuseChannelData>(&mount.channelInfo)) {
// Stuffing the fuse connection information in as a binary
// blob because we know that the endianness of the target
// machine must match the current system for a graceful
// takeover, and it saves us from re-encoding an operating
// system specific struct into a thrift file.
serializedMount.connInfo_ref() = std::string{
reinterpret_cast<const char*>(&fuseChannelInfo->connInfo),
sizeof(fuseChannelInfo->connInfo)};
}
*serializedMount.inodeMap_ref() = mount.inodeMap;
serializedMount.mountProtocol_ref() = mountProtocol;
serializedMounts.emplace_back(std::move(serializedMount));
}
@ -459,7 +578,9 @@ folly::IOBuf TakeoverData::serializeErrorThrift(
return std::move(*bufQ.move());
}
TakeoverData TakeoverData::deserializeThrift(IOBuf* buf) {
TakeoverData TakeoverData::deserializeThrift(
uint32_t protocolCapabilities,
IOBuf* buf) {
auto serialized = CompactSerializer::deserialize<SerializedTakeoverData>(buf);
switch (serialized.getType()) {
@ -469,21 +590,50 @@ TakeoverData TakeoverData::deserializeThrift(IOBuf* buf) {
case SerializedTakeoverData::Type::mounts: {
TakeoverData data;
for (auto& serializedMount : serialized.mutable_mounts()) {
const auto* connInfo = reinterpret_cast<const fuse_init_out*>(
serializedMount.connInfo_ref()->data());
std::vector<AbsolutePath> bindMounts;
for (const auto& path : *serializedMount.bindMountPaths_ref()) {
bindMounts.emplace_back(AbsolutePathPiece{path});
}
data.mountPoints.emplace_back(
AbsolutePath{*serializedMount.mountPath_ref()},
AbsolutePath{*serializedMount.stateDirectory_ref()},
std::move(bindMounts),
folly::File{},
*connInfo,
std::move(*serializedMount.inodeMap_ref()));
switch (*serializedMount.mountProtocol_ref()) {
case TakeoverMountProtocol::UNKNOWN:
if (protocolCapabilities & TakeoverCapabilities::MOUNT_TYPES) {
throw std::runtime_error("Unknown Mount Protocol");
}
// versions <5 all assumed FUSE mounts, but we don't want to make
// the default mount protocol fuse. We can fall through to parsing a
// fuse mount in this case.
[[fallthrough]];
case TakeoverMountProtocol::FUSE:
checkCanSerDeMountType(
protocolCapabilities,
TakeoverMountProtocol::FUSE,
*serializedMount.mountPath_ref());
data.mountPoints.emplace_back(
AbsolutePath{*serializedMount.mountPath_ref()},
AbsolutePath{*serializedMount.stateDirectory_ref()},
std::move(bindMounts),
FuseChannelData{
folly::File{},
*reinterpret_cast<const fuse_init_out*>(
serializedMount.connInfo_ref()->data())},
std::move(*serializedMount.inodeMap_ref()));
break;
case TakeoverMountProtocol::NFS:
checkCanSerDeMountType(
protocolCapabilities,
TakeoverMountProtocol::NFS,
*serializedMount.mountPath_ref());
data.mountPoints.emplace_back(
AbsolutePath{*serializedMount.mountPath_ref()},
AbsolutePath{*serializedMount.stateDirectory_ref()},
std::move(bindMounts),
NfsChannelData{folly::File{}},
std::move(*serializedMount.inodeMap_ref()));
break;
default:
throw std::runtime_error(
"impossible enum variant for TakeoverMountProtocol");
}
}
return data;
}

View File

@ -7,16 +7,18 @@
#pragma once
#include <folly/File.h>
#include <folly/futures/Promise.h>
#include <memory>
#include <optional>
#include <vector>
#include <folly/File.h>
#include <folly/futures/Promise.h>
#include "eden/fs/takeover/gen-cpp2/takeover_types.h"
#include "eden/fs/utils/FsChannelTypes.h"
#include "eden/fs/utils/FutureUnixSocket.h"
#include "eden/fs/utils/PathFuncs.h"
#include "eden/fs/utils/UnixSocket.h"
namespace folly {
class IOBuf;
@ -73,6 +75,14 @@ class TakeoverCapabilities {
// This should be used in all modern takeover versions.
PING = 1 << 3,
// Indicates that the protocol includes the type of kernel module that will
// be used for each mount point.
// This should be used in all modern takeover versions.
MOUNT_TYPES = 1 << 4,
// Indicates this version of the protocol is able to serialize NFS mount
// points
NFS = 1 << 5,
};
};
@ -108,7 +118,7 @@ class TakeoverData {
// version 2 to describe this next one.
kTakeoverProtocolVersionThree = 3,
// This version introduced an additonal handshake before taking over
// This version introduced an additional handshake before taking over
// that is sent after the TakeoverData is ready but before actually
// sending it. This is in order to make sure we only send the data if
// the new process is healthy and able to receive, because otherwise
@ -118,6 +128,13 @@ class TakeoverData {
// break a server with this extra handshake talking to a client
// without it
kTakeoverProtocolVersionFour = 4,
// This version introduced the ability to takeover NFS mounts.
// This includes serializing the mountd socket as well as the
// connected socket to the kernel for each of the mount points.
kTakeoverProtocolVersionFive = 5,
// version 5 should be the last real version, we should bump to version 6
// and from then on only match capabilities
};
/**
@ -147,41 +164,63 @@ class TakeoverData {
const std::set<int32_t>& supported = kSupportedTakeoverVersions);
struct MountInfo {
/**
* Constructor for an NFS mount's MountInfo
*/
MountInfo(
AbsolutePathPiece mountPath,
AbsolutePathPiece stateDirectory,
const std::vector<AbsolutePath>& bindMountPaths,
folly::File fd,
#ifndef _WIN32
fuse_init_out connInfo,
#endif
NfsChannelData nfsChannelData,
SerializedInodeMap&& inodeMap)
: mountPath{mountPath},
stateDirectory{stateDirectory},
bindMounts{bindMountPaths},
fuseFD{std::move(fd)},
#ifndef _WIN32
connInfo{connInfo},
#endif
inodeMap{std::move(inodeMap)} {
}
channelInfo{std::move(nfsChannelData)},
inodeMap{std::move(inodeMap)} {}
/**
* Constructor for a Fuse mount's MountInfo
*/
MountInfo(
AbsolutePathPiece mountPath,
AbsolutePathPiece stateDirectory,
const std::vector<AbsolutePath>& bindMountPaths,
FuseChannelData fuseChannelData,
SerializedInodeMap&& inodeMap)
: mountPath{mountPath},
stateDirectory{stateDirectory},
bindMounts{bindMountPaths},
channelInfo{std::move(fuseChannelData)},
inodeMap{std::move(inodeMap)} {}
/**
* Constructor for a Projected FS mount's MountInfo
*/
MountInfo(
AbsolutePathPiece mountPath,
AbsolutePathPiece stateDirectory,
const std::vector<AbsolutePath>& bindMountPaths,
ProjFsChannelData projfsChannelData,
SerializedInodeMap&& inodeMap)
: mountPath{mountPath},
stateDirectory{stateDirectory},
bindMounts{bindMountPaths},
channelInfo{std::move(projfsChannelData)},
inodeMap{std::move(inodeMap)} {}
AbsolutePath mountPath;
AbsolutePath stateDirectory;
std::vector<AbsolutePath> bindMounts;
folly::File fuseFD;
#ifndef _WIN32
fuse_init_out connInfo;
#endif
std::variant<FuseChannelData, NfsChannelData, ProjFsChannelData>
channelInfo;
SerializedInodeMap inodeMap;
};
/**
* Serialize the TakeoverData into a buffer that can be sent to a remote
* process.
*
* This includes all data except for file descriptors. The file descriptors
* must be sent separately.
* Serialize the TakeoverData into a unix socket message.
*/
void serialize(uint64_t protocolCapabilities, UnixSocket::Message& msg);
@ -216,6 +255,13 @@ class TakeoverData {
*/
static bool isPing(const folly::IOBuf* buf);
/**
* Determines if we should serialized NFS data given the protocol version
* we are serializing with. i.e. should we send takeover data for NFS mount
* points and should we send the mountd socket.
*/
static bool shouldSerdeNFSInfo(uint32_t protocolVersionCapabilies);
/**
* The main eden lock file that prevents two edenfs processes from running at
* the same time.
@ -227,6 +273,11 @@ class TakeoverData {
*/
folly::File thriftSocket;
/**
* Server socket for the mountd.
*/
folly::File mountdServerSocket;
/**
* The list of mount points.
*/
@ -240,7 +291,7 @@ class TakeoverData {
private:
/**
* Serialize the TakeoverData using the specified protocol version into a
* Serialize the TakeoverData using the specified protocol capabilities into a
* buffer that can be sent to a remote process.
*
* This includes all data except for file descriptors; these must be sent
@ -249,22 +300,26 @@ class TakeoverData {
folly::IOBuf serialize(uint64_t protocolCapabilities);
/**
* Serialize data using version 1 of the takeover protocol.
* Serialize data using version 1 of the takeover protocol. This uses a home
* grown serialization format.
*/
folly::IOBuf serializeCustom();
/**
* Serialize an exception using version 1 of the takeover protocol.
* Serialize an exception using version 1 of the takeover protocol. This uses
* a home grown serialization format
*/
static folly::IOBuf serializeErrorCustom(const folly::exception_wrapper& ew);
/**
* Serialize data using version 2 of the takeover protocol.
* Serialize data for any version that uses thrift serialization. This is
* versions 3+.
*/
folly::IOBuf serializeThrift(uint64_t protocolCapabilities);
/**
* Serialize an exception using version 2 of the takeover protocol.
* Serialize an exception for any version that uses thrift serialization. This
* is versions 3+.
*/
static folly::IOBuf serializeErrorThrift(const folly::exception_wrapper& ew);
@ -277,14 +332,16 @@ class TakeoverData {
folly::IOBuf* buf);
/**
* Deserialize the TakeoverData from a buffer using version 3 (also known as
* 2), 4 or 5 of the takeover protocol.
* Deserialize the TakeoverData from a buffer for any version of the protocol
* that uses thrift serialization. This is any version 3+.
*/
static TakeoverData deserializeThrift(folly::IOBuf* buf);
static TakeoverData deserializeThrift(
uint32_t protocolCapabilities,
folly::IOBuf* buf);
/**
* Deserialize the TakeoverData from a buffer using version 1 of the takeover
* protocol.
* protocol. This uses a home grown serialization format.
*/
static TakeoverData deserializeCustom(folly::IOBuf* buf);

View File

@ -48,8 +48,13 @@ namespace eden {
*/
class TakeoverServer::ConnHandler {
public:
ConnHandler(TakeoverServer* server, folly::File socket)
: server_{server}, socket_{server_->getEventBase(), std::move(socket)} {}
ConnHandler(
TakeoverServer* server,
folly::File socket,
const std::set<int32_t>& supportedVersions)
: server_{server},
socket_{server_->getEventBase(), std::move(socket)},
supportedVersions_{supportedVersions} {}
/**
* start() begins processing data on this connection.
@ -79,6 +84,7 @@ class TakeoverServer::ConnHandler {
bool shouldPing_{false};
TakeoverServer* const server_{nullptr};
FutureUnixSocket socket_;
const std::set<int32_t>& supportedVersions_;
int32_t protocolVersion_{
TakeoverData::kTakeoverProtocolVersionNeverSupported};
uint64_t protocolCapabilities_{0};
@ -126,13 +132,13 @@ Future<Unit> TakeoverServer::ConnHandler::start() noexcept {
auto query =
CompactSerializer::deserialize<TakeoverVersionQuery>(&msg->data);
auto supported =
TakeoverData::computeCompatibleVersion(*query.versions_ref());
auto supported = TakeoverData::computeCompatibleVersion(
*query.versions_ref(), this->supportedVersions_);
if (!supported.has_value()) {
auto clientVersionList = folly::join(", ", *query.versions_ref());
auto serverVersionList =
folly::join(", ", kSupportedTakeoverVersions);
folly::join(", ", this->supportedVersions_);
return folly::makeFuture<TakeoverData>(
folly::make_exception_wrapper<std::runtime_error>(
@ -151,6 +157,9 @@ Future<Unit> TakeoverServer::ConnHandler::start() noexcept {
protocolVersion_ = supported.value();
protocolCapabilities_ =
TakeoverData::versionToCapabilites(protocolVersion_);
XLOG(DBG7) << "Protocol version: " << protocolVersion_
<< "; Protocol Capabilities: " << protocolCapabilities_;
shouldPing_ = (protocolCapabilities_ & TakeoverCapabilities::PING);
return server_->getTakeoverHandler()->startTakeoverShutdown();
})
@ -267,11 +276,13 @@ TakeoverServer::TakeoverServer(
folly::EventBase* eventBase,
AbsolutePathPiece socketPath,
TakeoverHandler* handler,
FaultInjector* faultInjector)
FaultInjector* faultInjector,
const std::set<int32_t>& supportedVersions)
: eventBase_{eventBase},
handler_{handler},
socketPath_{socketPath},
faultInjector_(*faultInjector) {
faultInjector_(*faultInjector),
supportedVersions_{supportedVersions} {
start();
}
@ -304,7 +315,7 @@ void TakeoverServer::connectionAccepted(
folly::File socket(fd, /* ownsFd */ true);
std::unique_ptr<ConnHandler> handler;
try {
handler.reset(new ConnHandler{this, std::move(socket)});
handler.reset(new ConnHandler{this, std::move(socket), supportedVersions_});
} catch (const std::exception& ex) {
XLOG(ERR) << "error allocating connection handler for new takeover "
"connection: "

View File

@ -10,6 +10,7 @@
#include <folly/io/async/AsyncServerSocket.h>
#include <memory>
#include "eden/fs/takeover/TakeoverData.h"
#include "eden/fs/utils/FaultInjector.h"
#include "eden/fs/utils/PathFuncs.h"
@ -29,7 +30,8 @@ class TakeoverServer : private folly::AsyncServerSocket::AcceptCallback {
folly::EventBase* eventBase,
AbsolutePathPiece socketPath,
TakeoverHandler* handler,
FaultInjector* FOLLY_NONNULL faultInjector);
FaultInjector* FOLLY_NONNULL faultInjector,
const std::set<int32_t>& supportedVersions = kSupportedTakeoverVersions);
virtual ~TakeoverServer() override;
void start();
@ -59,6 +61,10 @@ class TakeoverServer : private folly::AsyncServerSocket::AcceptCallback {
AbsolutePath socketPath_;
folly::AsyncServerSocket::UniquePtr socket_;
FaultInjector& faultInjector_;
// generally this should be kSupportedTakeoverVersions, but we allow setting
// it differently, mostly for tests so that you can test a version that might
// not be ready for production yet
const std::set<int32_t>& supportedVersions_;
};
} // namespace eden
} // namespace facebook

View File

@ -29,6 +29,13 @@ struct SerializedInodeMap {
struct SerializedFileHandleMap {}
// Mount protocol to use for a mount that we are taking over.
enum TakeoverMountProtocol {
UNKNOWN = 0,
FUSE = 1,
NFS = 2,
}
struct SerializedMountInfo {
1: string mountPath;
2: string stateDirectory;
@ -41,12 +48,15 @@ struct SerializedMountInfo {
// access the struct once we've moved it across the process
// boundary. Note that takeover is always local to the same
// machine and thus has the same endianness.
// This will be left empty for NFS mounts.
4: binary connInfo; // fuse_init_out
// Removed, do not use 5
// 5: SerializedFileHandleMap fileHandleMap,
6: SerializedInodeMap inodeMap;
7: TakeoverMountProtocol mountProtocol = TakeoverMountProtocol.UNKNOWN;
}
union SerializedTakeoverData {

View File

@ -117,7 +117,9 @@ void loopWithTimeout(EventBase* evb, std::chrono::milliseconds timeout = 300s) {
folly::Try<TakeoverData> runTakeover(
const TemporaryDirectory& tmpDir,
TakeoverHandler* handler,
const std::set<int32_t>& supportedVersions = kSupportedTakeoverVersions) {
const std::set<int32_t>& supportedVersions = kSupportedTakeoverVersions,
const std::set<int32_t>& serverSupportedVersions =
kSupportedTakeoverVersions) {
// Ignore SIGPIPE so that sendmsg() will fail with an error code instead
// of terminating the program if the remote side has closed the connection.
signal(SIGPIPE, SIG_IGN);
@ -127,7 +129,8 @@ folly::Try<TakeoverData> runTakeover(
EventBase evb;
FaultInjector faultInjector{/*enabled=*/false};
TakeoverServer server(&evb, socketPath, handler, &faultInjector);
TakeoverServer server(
&evb, socketPath, handler, &faultInjector, serverSupportedVersions);
auto future =
takeoverViaEventBase(&evb, socketPath, supportedVersions).ensure([&] {
@ -195,6 +198,10 @@ TEST(Takeover, simple) {
serverData.thriftSocket =
folly::File{thriftSocketPath.stringPiece(), O_RDWR | O_CREAT};
auto mountdSocketPath = tmpDirPath + "mountd"_pc;
serverData.mountdServerSocket =
folly::File{mountdSocketPath.stringPiece(), O_RDWR | O_CREAT};
auto mount1Path = tmpDirPath + "mount1"_pc;
auto client1Path = tmpDirPath + "client1"_pc;
auto mount1FusePath = tmpDirPath + "fuse1"_pc;
@ -202,8 +209,9 @@ TEST(Takeover, simple) {
mount1Path,
client1Path,
std::vector<AbsolutePath>{},
folly::File{mount1FusePath.stringPiece(), O_RDWR | O_CREAT},
fuse_init_out{},
FuseChannelData{
folly::File{mount1FusePath.stringPiece(), O_RDWR | O_CREAT},
fuse_init_out{}},
SerializedInodeMap{});
auto mount2Path = tmpDirPath + "mount2"_pc;
@ -218,8 +226,9 @@ TEST(Takeover, simple) {
mount2Path,
client2Path,
mount2BindMounts,
folly::File{mount2FusePath.stringPiece(), O_RDWR | O_CREAT},
fuse_init_out{},
FuseChannelData{
folly::File{mount2FusePath.stringPiece(), O_RDWR | O_CREAT},
fuse_init_out{}},
SerializedInodeMap{});
// Perform the takeover
@ -240,14 +249,18 @@ TEST(Takeover, simple) {
EXPECT_EQ(mount1Path, clientData.mountPoints.at(0).mountPath);
EXPECT_EQ(client1Path, clientData.mountPoints.at(0).stateDirectory);
EXPECT_THAT(clientData.mountPoints.at(0).bindMounts, ElementsAre());
checkExpectedFile(clientData.mountPoints.at(0).fuseFD.fd(), mount1FusePath);
auto& fuseChannelData0 =
std::get<FuseChannelData>(clientData.mountPoints.at(0).channelInfo);
checkExpectedFile(fuseChannelData0.fd.fd(), mount1FusePath);
EXPECT_EQ(mount2Path, clientData.mountPoints.at(1).mountPath);
EXPECT_EQ(client2Path, clientData.mountPoints.at(1).stateDirectory);
EXPECT_THAT(
clientData.mountPoints.at(1).bindMounts,
ElementsAreArray(mount2BindMounts));
checkExpectedFile(clientData.mountPoints.at(1).fuseFD.fd(), mount2FusePath);
auto& fuseChannelData1 =
std::get<FuseChannelData>(clientData.mountPoints.at(1).channelInfo);
checkExpectedFile(fuseChannelData1.fd.fd(), mount2FusePath);
}
TEST(Takeover, noMounts) {
@ -262,6 +275,9 @@ TEST(Takeover, noMounts) {
auto thriftSocketPath = tmpDirPath + "thrift"_pc;
serverData.thriftSocket =
folly::File{thriftSocketPath.stringPiece(), O_RDWR | O_CREAT};
auto mountdSocketPath = tmpDirPath + "mountd"_pc;
serverData.mountdServerSocket =
folly::File{mountdSocketPath.stringPiece(), O_RDWR | O_CREAT};
// Perform the takeover
auto serverSendFuture = serverData.takeoverComplete.getFuture();
@ -292,6 +308,9 @@ TEST(Takeover, manyMounts) {
auto thriftSocketPath = tmpDirPath + "thrift"_pc;
serverData.thriftSocket =
folly::File{thriftSocketPath.stringPiece(), O_RDWR | O_CREAT};
auto mountdSocketPath = tmpDirPath + "mountd"_pc;
serverData.mountdServerSocket =
folly::File{mountdSocketPath.stringPiece(), O_RDWR | O_CREAT};
// Build info for 10,000 mounts
// This exercises the code where we send more FDs than ControlMsg::kMaxFDs.
@ -318,8 +337,9 @@ TEST(Takeover, manyMounts) {
mountPath,
stateDirectory,
bindMounts,
folly::File{fusePath.stringPiece(), O_RDWR | O_CREAT},
fuse_init_out{},
FuseChannelData{
folly::File{fusePath.stringPiece(), O_RDWR | O_CREAT},
fuse_init_out{}},
SerializedInodeMap{});
}
@ -357,7 +377,8 @@ TEST(Takeover, manyMounts) {
auto expectedFusePath =
tmpDirPath + PathComponentPiece{folly::to<string>("fuse", n)};
checkExpectedFile(mountInfo.fuseFD.fd(), expectedFusePath);
auto& fuseChannelData = std::get<FuseChannelData>(mountInfo.channelInfo);
checkExpectedFile(fuseChannelData.fd.fd(), expectedFusePath);
}
}
@ -444,6 +465,9 @@ TEST(Takeover, oneToTwo) {
auto thriftSocketPath = tmpDirPath + "thrift"_pc;
serverData.thriftSocket =
folly::File{thriftSocketPath.stringPiece(), O_RDWR | O_CREAT};
auto mountdSocketPath = tmpDirPath + "mountd"_pc;
serverData.mountdServerSocket =
folly::File{mountdSocketPath.stringPiece(), O_RDWR | O_CREAT};
// Perform the takeover, explicitly using the older version
// of the takeover protocol
@ -461,7 +485,147 @@ TEST(Takeover, oneToTwo) {
// expected files.
checkExpectedFile(clientData.lockFile.fd(), lockFilePath);
checkExpectedFile(clientData.thriftSocket.fd(), thriftSocketPath);
// version 1 and 2 are not able to receive the mountd socket
EXPECT_EQ(clientData.mountdServerSocket.fd(), -1);
// Make sure the received mount information is empty
EXPECT_EQ(0, clientData.mountPoints.size());
}
TEST(Takeover, nfs) {
TemporaryDirectory tmpDir("eden_takeover_test");
AbsolutePathPiece tmpDirPath{tmpDir.path().string()};
// Build the TakeoverData object to send
TakeoverData serverData;
auto lockFilePath = tmpDirPath + "lock"_pc;
serverData.lockFile =
folly::File{lockFilePath.stringPiece(), O_RDWR | O_CREAT};
auto thriftSocketPath = tmpDirPath + "thrift"_pc;
serverData.thriftSocket =
folly::File{thriftSocketPath.stringPiece(), O_RDWR | O_CREAT};
auto mountdSocketPath = tmpDirPath + "mountd"_pc;
serverData.mountdServerSocket =
folly::File{mountdSocketPath.stringPiece(), O_RDWR | O_CREAT};
auto mount1Path = tmpDirPath + "mount1"_pc;
auto client1Path = tmpDirPath + "client1"_pc;
auto mount1FusePath = tmpDirPath + "fuse1"_pc;
serverData.mountPoints.emplace_back(
mount1Path,
client1Path,
std::vector<AbsolutePath>{},
FuseChannelData{
folly::File{mount1FusePath.stringPiece(), O_RDWR | O_CREAT},
fuse_init_out{}},
SerializedInodeMap{});
auto mount2Path = tmpDirPath + "mount2"_pc;
auto client2Path = tmpDirPath + "client2"_pc;
auto mount2NfsPath = tmpDirPath + "nfs"_pc;
std::vector<AbsolutePath> mount2BindMounts = {
mount2Path + "test/test2"_relpath,
AbsolutePath{"/foo/bar"},
mount2Path + "a/b/c/d/e/f"_relpath,
};
serverData.mountPoints.emplace_back(
mount2Path,
client2Path,
mount2BindMounts,
NfsChannelData{
folly::File{mount2NfsPath.stringPiece(), O_RDWR | O_CREAT}},
SerializedInodeMap{});
// Perform the takeover
auto serverSendFuture = serverData.takeoverComplete.getFuture();
TestHandler handler{std::move(serverData)};
auto result = runTakeover(
tmpDir,
&handler,
std::set<int32_t>{
TakeoverData::kTakeoverProtocolVersionFour,
TakeoverData::kTakeoverProtocolVersionFive},
std::set<int32_t>{
TakeoverData::kTakeoverProtocolVersionFour,
TakeoverData::kTakeoverProtocolVersionFive});
ASSERT_TRUE(serverSendFuture.hasValue());
EXPECT_TRUE(result.hasValue());
const auto& clientData = result.value();
// Make sure the received lock file refers to the expected file.
checkExpectedFile(clientData.lockFile.fd(), lockFilePath);
// And the thrift socket FD
checkExpectedFile(clientData.thriftSocket.fd(), thriftSocketPath);
checkExpectedFile(clientData.mountdServerSocket.fd(), mountdSocketPath);
// Make sure the received mount information is correct
ASSERT_EQ(2, clientData.mountPoints.size());
EXPECT_EQ(mount1Path, clientData.mountPoints.at(0).mountPath);
EXPECT_EQ(client1Path, clientData.mountPoints.at(0).stateDirectory);
EXPECT_THAT(clientData.mountPoints.at(0).bindMounts, ElementsAre());
auto& fuseChannelData =
std::get<FuseChannelData>(clientData.mountPoints.at(0).channelInfo);
checkExpectedFile(fuseChannelData.fd.fd(), mount1FusePath);
EXPECT_EQ(mount2Path, clientData.mountPoints.at(1).mountPath);
EXPECT_EQ(client2Path, clientData.mountPoints.at(1).stateDirectory);
EXPECT_THAT(
clientData.mountPoints.at(1).bindMounts,
ElementsAreArray(mount2BindMounts));
auto& nfsChannelData =
std::get<NfsChannelData>(clientData.mountPoints.at(1).channelInfo);
checkExpectedFile(nfsChannelData.nfsdSocketFd.fd(), mount2NfsPath);
}
TEST(Takeover, nfsOldVersion) {
TemporaryDirectory tmpDir("eden_takeover_test");
AbsolutePathPiece tmpDirPath{tmpDir.path().string()};
// Build the TakeoverData object to send
TakeoverData serverData;
auto lockFilePath = tmpDirPath + "lock"_pc;
serverData.lockFile =
folly::File{lockFilePath.stringPiece(), O_RDWR | O_CREAT};
auto thriftSocketPath = tmpDirPath + "thrift"_pc;
serverData.thriftSocket =
folly::File{thriftSocketPath.stringPiece(), O_RDWR | O_CREAT};
auto mountdSocketPath = tmpDirPath + "mountd"_pc;
serverData.mountdServerSocket =
folly::File{mountdSocketPath.stringPiece(), O_RDWR | O_CREAT};
auto mountPath = tmpDirPath + "mount2"_pc;
auto clientPath = tmpDirPath + "client2"_pc;
auto mountNfsPath = tmpDirPath + "nfs"_pc;
std::vector<AbsolutePath> mountBindMounts = {
mountPath + "test/test2"_relpath,
AbsolutePath{"/foo/bar"},
mountPath + "a/b/c/d/e/f"_relpath,
};
serverData.mountPoints.emplace_back(
mountPath,
clientPath,
mountBindMounts,
NfsChannelData{folly::File{mountNfsPath.stringPiece(), O_RDWR | O_CREAT}},
SerializedInodeMap{});
// Perform the takeover
auto serverSendFuture = serverData.takeoverComplete.getFuture();
TestHandler handler{std::move(serverData)};
auto result = runTakeover(
tmpDir,
&handler,
std::set<int32_t>{TakeoverData::kTakeoverProtocolVersionFour});
EXPECT_TRUE(result.hasException());
EXPECT_THROW_RE(
result.exception().throw_exception(),
std::runtime_error,
"protocol does not support serializing/deserializing this type of "
"mounts. protocol capabilities: 14. problem mount: .*mount2. mount "
"protocol: 2");
}

View File

@ -33,4 +33,8 @@ struct NfsChannelData {
folly::File nfsdSocketFd;
};
struct ProjFsChannelData {
// TODO fill this in with data to support takeover on windows
};
} // namespace facebook::eden

View File

@ -13,6 +13,7 @@
#include <gflags/gflags.h>
#include "eden/fs/takeover/TakeoverClient.h"
#include "eden/fs/takeover/TakeoverData.h"
#include "eden/fs/utils/FsChannelTypes.h"
DEFINE_string(edenDir, "", "The path to the .eden directory");
/**
@ -59,7 +60,16 @@ int main(int argc, char* argv[]) {
takeoverSocketPath, FLAGS_shouldPing, takeoverVersion);
}
for (const auto& mount : data.mountPoints) {
XLOG(INFO) << "mount " << mount.mountPath << ": fd=" << mount.fuseFD.fd();
const folly::File* mountFD = nullptr;
if (auto fuseChannelData =
std::get_if<facebook::eden::FuseChannelData>(&mount.channelInfo)) {
mountFD = &fuseChannelData->fd;
} else {
auto& nfsChannelData =
std::get<facebook::eden::NfsChannelData>(mount.channelInfo);
mountFD = &nfsChannelData.nfsdSocketFd;
}
XLOG(INFO) << "mount " << mount.mountPath << ": fd=" << mountFD->fd();
for (const auto& bindMount : mount.bindMounts) {
XLOG(INFO) << " bind mount " << bindMount;
}