diff --git a/eden/fs/inodes/EdenMount.cpp b/eden/fs/inodes/EdenMount.cpp index d8942974eb..f2fa0e794a 100644 --- a/eden/fs/inodes/EdenMount.cpp +++ b/eden/fs/inodes/EdenMount.cpp @@ -219,13 +219,15 @@ EdenMount::EdenMount( } FOLLY_NODISCARD folly::Future EdenMount::initialize( + std::function&& progressCallback, const std::optional& takeover) { transitionState(State::UNINITIALIZED, State::INITIALIZING); return serverState_->getFaultInjector() .checkAsync("mount", getPath().stringPiece()) .via(serverState_->getThreadPool().get()) - .thenValue([this](auto&&) { + .thenValue([this, progressCallback = std::move(progressCallback)]( + auto&&) mutable { auto parents = config_->getParentCommits(); parentInfo_.wlock()->parents.setParents(parents); @@ -235,10 +237,10 @@ FOLLY_NODISCARD folly::Future EdenMount::initialize( journal_->recordHashUpdate(parents.parent1()); // Initialize the overlay. - // This must be performed before we do any operations that may allocate - // inode numbers, including creating the root TreeInode. - return overlay_->initialize().deferValue( - [parents](auto&&) { return parents; }); + // This must be performed before we do any operations that may + // allocate inode numbers, including creating the root TreeInode. + return overlay_->initialize(std::move(progressCallback)) + .deferValue([parents](auto&&) { return parents; }); }) .thenValue( [this](ParentCommits&& parents) { return createRootInode(parents); }) @@ -837,7 +839,6 @@ folly::Future EdenMount::checkout( *lastCheckoutTime_.wlock() = clock_->getRealtime(); auto journalDiffCallback = std::make_shared(); - return serverState_->getFaultInjector() .checkAsync("checkout", getPath().stringPiece()) .via(serverState_->getThreadPool().get()) diff --git a/eden/fs/inodes/EdenMount.h b/eden/fs/inodes/EdenMount.h index 3d8c421af5..b065b4a4cd 100644 --- a/eden/fs/inodes/EdenMount.h +++ b/eden/fs/inodes/EdenMount.h @@ -176,6 +176,7 @@ class EdenMount { * If takeover data is specified, it is used to initialize the inode map. */ FOLLY_NODISCARD folly::Future initialize( + std::function&& progressCallback = nullptr, const std::optional& takeover = std::nullopt); /** diff --git a/eden/fs/inodes/Overlay.cpp b/eden/fs/inodes/Overlay.cpp index 4bdffc8388..af16a3e51a 100644 --- a/eden/fs/inodes/Overlay.cpp +++ b/eden/fs/inodes/Overlay.cpp @@ -103,7 +103,8 @@ struct statfs Overlay::statFs() { } #endif // !_WIN32 -folly::SemiFuture Overlay::initialize() { +folly::SemiFuture Overlay::initialize( + std::function&& progressCallback) { // The initOverlay() call is potentially slow, so we want to avoid // performing it in the current thread and blocking returning to our caller. // @@ -111,9 +112,12 @@ folly::SemiFuture Overlay::initialize() { // to simply use this existing thread to perform the initialization logic // before waiting for GC work to do. auto [initPromise, initFuture] = folly::makePromiseContract(); - gcThread_ = std::thread([this, promise = std::move(initPromise)]() mutable { + + gcThread_ = std::thread([this, + progressCallback = std::move(progressCallback), + promise = std::move(initPromise)]() mutable { try { - initOverlay(); + initOverlay(progressCallback); } catch (std::exception& ex) { XLOG(ERR) << "overlay initialization failed for " << backingOverlay_.getLocalDir() << ": " << ex.what(); @@ -123,8 +127,8 @@ folly::SemiFuture Overlay::initialize() { } promise.setValue(); #ifndef _WIN32 - // TODO: On Windows files are cached by the ProjectedFS. We need to clean - // the cached files while doing GC. + // TODO: On Windows files are cached by the ProjectedFS. We need to + // clean the cached files while doing GC. gcThread(); #endif @@ -132,7 +136,8 @@ folly::SemiFuture Overlay::initialize() { return std::move(initFuture); } -void Overlay::initOverlay() { +void Overlay::initOverlay( + const std::function& progressCallback) { IORequest req{this}; auto optNextInodeNumber = backingOverlay_.initOverlay(true); if (!optNextInodeNumber.has_value()) { @@ -146,8 +151,9 @@ void Overlay::initOverlay() { // correct next inode number as it does so. XLOG(WARN) << "Overlay " << backingOverlay_.getLocalDir() << " was not shut down cleanly. Performing fsck scan."; + OverlayChecker checker(&backingOverlay_, std::nullopt); - checker.scanForErrors(); + checker.scanForErrors(progressCallback); checker.repairErrors(); optNextInodeNumber = checker.getNextInodeNumber(); @@ -208,7 +214,8 @@ optional Overlay::loadOverlayDir(InodeNumber inodeNumber) { const auto& name = iter.first; const auto& value = iter.second; - bool isMaterialized = !value.hash_ref() || value.hash_ref().value().empty(); + bool isMaterialized = + !value.hash_ref() || value.hash_ref().value_unchecked().empty(); InodeNumber ino; if (value.inodeNumber) { ino = InodeNumber::fromThrift(value.inodeNumber); diff --git a/eden/fs/inodes/Overlay.h b/eden/fs/inodes/Overlay.h index 5291e74f7c..041b6a03c7 100644 --- a/eden/fs/inodes/Overlay.h +++ b/eden/fs/inodes/Overlay.h @@ -100,7 +100,8 @@ class Overlay : public std::enable_shared_from_this { * - Upgrading the on-disk data from older formats if the Overlay was created * by an older version of the software. */ - FOLLY_NODISCARD folly::SemiFuture initialize(); + FOLLY_NODISCARD folly::SemiFuture initialize( + std::function&& progressCallback = nullptr); /** * Closes the overlay. It is undefined behavior to access the @@ -243,7 +244,8 @@ class Overlay : public std::enable_shared_from_this { std::vector queue; }; - void initOverlay(); + void initOverlay( + const std::function& progressCallback = nullptr); void gcThread() noexcept; void handleGCRequest(GCRequest& request); diff --git a/eden/fs/inodes/overlay/OverlayChecker.cpp b/eden/fs/inodes/overlay/OverlayChecker.cpp index cc5dcfac8c..6c59713877 100644 --- a/eden/fs/inodes/overlay/OverlayChecker.cpp +++ b/eden/fs/inodes/overlay/OverlayChecker.cpp @@ -655,9 +655,16 @@ OverlayChecker::OverlayChecker( OverlayChecker::~OverlayChecker() {} -void OverlayChecker::scanForErrors() { - XLOG(INFO) << "Starting fsck scan on overlay " << fs_->getLocalDir(); - readInodes(); +void OverlayChecker::scanForErrors( + const std::function& progressCallback) { + auto fsck_start_msg = folly::to( + "Starting fsck scan on overlay ", fs_->getLocalDir()); + XLOG(INFO) << fsck_start_msg; + + if (auto callback = progressCallback) { + callback(fsck_start_msg); + } + readInodes(progressCallback); linkInodeChildren(); scanForParentErrors(); checkNextInodeNumber(); @@ -845,7 +852,8 @@ PathComponent OverlayChecker::findChildName( return PathComponent(folly::to("[missing_child(", child, ")]")); } -void OverlayChecker::readInodes() { +void OverlayChecker::readInodes( + const std::function& progressCallback) { // Walk through all of the sharded subdirectories uint32_t progress10pct = 0; std::array subdirBuffer; @@ -854,8 +862,18 @@ void OverlayChecker::readInodes() { // Log a DBG2 message every 10% done uint32_t progress = (10 * shardID) / FsOverlay::kNumShards; if (progress > progress10pct) { - XLOG(DBG2) << "fsck:" << fs_->getLocalDir() << ": scan " << progress - << "0% complete: " << inodes_.size() << " inodes scanned"; + auto fsck_path_msg = + folly::to("fsck:", fs_->getLocalDir(), ": "); + auto fsck_scanned_msg = folly::to( + "scan ", + progress, + "0% complete: ", + inodes_.size(), + " inodes scanned"); + XLOG(INFO) << fsck_path_msg << fsck_scanned_msg; + if (auto callback = progressCallback) { + callback(fsck_scanned_msg); + } progress10pct = progress; } diff --git a/eden/fs/inodes/overlay/OverlayChecker.h b/eden/fs/inodes/overlay/OverlayChecker.h index d609e77991..1754cf8693 100644 --- a/eden/fs/inodes/overlay/OverlayChecker.h +++ b/eden/fs/inodes/overlay/OverlayChecker.h @@ -64,7 +64,8 @@ class OverlayChecker { /** * Scan the overlay for problems. */ - void scanForErrors(); + void scanForErrors( + const std::function& progressCallback = nullptr); /** * Attempt to repair the errors that were found by scanForErrors(). @@ -185,7 +186,8 @@ class OverlayChecker { PathInfo cachedPathComputation(InodeNumber number, Fn&& fn); using ShardID = uint32_t; - void readInodes(); + void readInodes( + const std::function& progressCallback = nullptr); void readInodeSubdir(const AbsolutePath& path, ShardID shardID); void loadInode(InodeNumber number, ShardID shardID); InodeInfo loadInodeInfo(InodeNumber number); diff --git a/eden/fs/service/EdenServer.cpp b/eden/fs/service/EdenServer.cpp index e4f6b3cb25..1cd72fe0dc 100644 --- a/eden/fs/service/EdenServer.cpp +++ b/eden/fs/service/EdenServer.cpp @@ -852,7 +852,12 @@ std::vector> EdenServer::prepareMountsTakeover( auto initialConfig = CheckoutConfig::loadFromClientDirectory( AbsolutePathPiece{info.mountPath}, AbsolutePathPiece{info.stateDirectory}); - return mount(std::move(initialConfig), false, std::move(info)); + + return mount( + std::move(initialConfig), + false, + [logger](auto msg) { logger->log(msg); }, + std::move(info)); }) .thenTry([logger, mountPath = info.mountPath]( folly::Try>&& result) { @@ -911,7 +916,10 @@ std::vector> EdenServer::prepareMounts( auto initialConfig = CheckoutConfig::loadFromClientDirectory( AbsolutePathPiece{mountInfo.mountPoint}, AbsolutePathPiece{mountInfo.edenClientPath}); - return mount(std::move(initialConfig), false); + + return mount(std::move(initialConfig), false, [logger](auto msg) { + logger->log(msg); + }); }) .thenTry([logger, mountPath = client.first.asString()]( folly::Try>&& result) { @@ -1196,6 +1204,7 @@ Future EdenServer::completeTakeoverStart( folly::Future> EdenServer::mount( std::unique_ptr initialConfig, bool readOnly, + std::function&& progressCallback, optional&& optionalTakeover) { folly::stop_watch<> mountStopWatch; @@ -1220,7 +1229,9 @@ folly::Future> EdenServer::mount( registerStats(edenMount); const bool doTakeover = optionalTakeover.has_value(); + auto initFuture = edenMount->initialize( + std::move(progressCallback), doTakeover ? std::make_optional(optionalTakeover->inodeMap) : std::nullopt); diff --git a/eden/fs/service/EdenServer.h b/eden/fs/service/EdenServer.h index efa21c72aa..69babdd14e 100644 --- a/eden/fs/service/EdenServer.h +++ b/eden/fs/service/EdenServer.h @@ -31,6 +31,7 @@ #include "eden/fs/inodes/ServerState.h" #include "eden/fs/service/EdenStateDir.h" #include "eden/fs/service/PeriodicTask.h" +#include "eden/fs/service/StartupLogger.h" #include "eden/fs/takeover/TakeoverHandler.h" #include "eden/fs/telemetry/EdenStats.h" #include "eden/fs/telemetry/RequestMetricsScope.h" @@ -220,6 +221,7 @@ class EdenServer : private TakeoverHandler { FOLLY_NODISCARD folly::Future> mount( std::unique_ptr initialConfig, bool readOnly, + std::function&& progressCallback = nullptr, std::optional&& optionalTakeover = std::nullopt); /** diff --git a/eden/fs/service/EdenServiceHandler.cpp b/eden/fs/service/EdenServiceHandler.cpp index 888db88110..8e0d1a6444 100644 --- a/eden/fs/service/EdenServiceHandler.cpp +++ b/eden/fs/service/EdenServiceHandler.cpp @@ -333,6 +333,7 @@ void EdenServiceHandler::mount(std::unique_ptr argument) { auto initialConfig = CheckoutConfig::loadFromClientDirectory( AbsolutePathPiece{argument->mountPoint}, AbsolutePathPiece{argument->edenClientPath}); + server_->mount(std::move(initialConfig), argument->readOnly).get(); } catch (const EdenError& ex) { XLOG(ERR) << "Error: " << ex.what(); diff --git a/eden/fs/testharness/TestMount.cpp b/eden/fs/testharness/TestMount.cpp index 2398aed889..3232079eed 100644 --- a/eden/fs/testharness/TestMount.cpp +++ b/eden/fs/testharness/TestMount.cpp @@ -341,7 +341,7 @@ void TestMount::remountGracefully() { blobCache_, serverState_, std::move(journal)); - edenMount_->initialize(takeoverData).getVia(serverExecutor_.get()); + edenMount_->initialize(nullptr, takeoverData).getVia(serverExecutor_.get()); } #endif