Expose TreeLookupCallback to OverlayChecker

Summary:
In a future diff we'll want the OverlayChecker to fix up the overlay by
dematerializing missing nodes, instead of creating empty files. To do that,
we'll need the ability to lookup the scm hash of a directory entry. This first
diff threads a lookup function from EdenMount down to the OverlayChecker. A
later diff will then use this callback.

Reviewed By: xavierd

Differential Revision: D37188336

fbshipit-source-id: 003143391de0fcf1b02ecc0c1336c0b7e88ec5d9

eden/fs/inodes/EdenMount.cpp

@@ -284,6 +284,66 @@ Overlay::OverlayType EdenMount::getOverlayType(
   }
 }
 
+namespace {
+
+class TreeLookupProcessor {
+ public:
+  explicit TreeLookupProcessor(
+      RelativePathPiece path,
+      std::shared_ptr<ObjectStore> objectStore,
+      ObjectFetchContext& context)
+      : path_{path},
+        iterRange_{path_.components()},
+        iter_{iterRange_.begin()},
+        objectStore_{std::move(objectStore)},
+        context_{context} {}
+
+  ImmediateFuture<OverlayChecker::LookupCallbackValue> next(
+      std::shared_ptr<const Tree> tree) {
+    using RetType = OverlayChecker::LookupCallbackValue;
+    auto name = *iter_++;
+    auto it = tree->find(name);
+
+    if (it == tree->cend()) {
+      return makeImmediateFuture<RetType>(
+          std::system_error(ENOENT, std::generic_category()));
+    }
+
+    if (iter_ == iterRange_.end()) {
+      if (it->second.isTree()) {
+        return objectStore_->getTree(it->second.getHash(), context_)
+            .thenValue([](std::shared_ptr<const Tree> tree) -> RetType {
+              return tree;
+            });
+      } else {
+        return ImmediateFuture{RetType{it->second}};
+      }
+    } else {
+      if (!it->second.isTree()) {
+        return makeImmediateFuture<RetType>(
+            std::system_error(ENOTDIR, std::generic_category()));
+      } else {
+        return objectStore_->getTree(it->second.getHash(), context_)
+            .thenValue([this](std::shared_ptr<const Tree> tree) {
+              return next(std::move(tree));
+            });
+      }
+    }
+  }
+
+ private:
+  RelativePath path_;
+  RelativePath::base_type::component_iterator_range iterRange_;
+  RelativePath::base_type::component_iterator iter_;
+  std::shared_ptr<ObjectStore> objectStore_;
+  // The ObjectFetchContext is allocated at the beginning of a request and
+  // released once the request completes. Since the lifetime of
+  // TreeLookupProcessor is strictly less than the one of a request, we can
+  // safely store a reference to the fetch context.
+  ObjectFetchContext& context_;
+};
+} // namespace
+
 FOLLY_NODISCARD folly::Future<folly::Unit> EdenMount::initialize(
     OverlayChecker::ProgressCallback&& progressCallback,
     const std::optional<SerializedInodeMap>& takeover) {
@@ -294,12 +354,12 @@ FOLLY_NODISCARD folly::Future<folly::Unit> EdenMount::initialize(
       parentCommit.getLastCheckoutId(ParentCommit::RootIdPreference::To)
           .value();
 
+  static auto context = ObjectFetchContext::getNullContextWithCauseDetail(
+      "EdenMount::initialize");
   return serverState_->getFaultInjector()
       .checkAsync("mount", getPath().stringPiece())
       .via(getServerThreadPool().get())
       .thenValue([this, parent](auto&&) {
-        static auto context = ObjectFetchContext::getNullContextWithCauseDetail(
-            "EdenMount::initialize");
         return objectStore_->getRootTree(parent, *context)
             .semi()
             .via(&folly::QueuedImmediateExecutor::instance());
@@ -338,7 +398,22 @@ FOLLY_NODISCARD folly::Future<folly::Unit> EdenMount::initialize(
         // Initialize the overlay.
         // This must be performed before we do any operations that may
         // allocate inode numbers, including creating the root TreeInode.
-        return overlay_->initialize(getPath(), std::move(progressCallback))
+        return overlay_
+            ->initialize(
+                getPath(),
+                std::move(progressCallback),
+                [this](RelativePathPiece path) {
+                  auto lookup = std::make_unique<TreeLookupProcessor>(
+                      path, objectStore_, *context);
+                  // Do the next() and the ensure() on separate lines to make
+                  // the order of 'lookup' accesses explicit, so we don't move
+                  // it before calling next.
+                  auto future = lookup->next(getCheckedOutRootTree());
+                  // The 'ensure' makes sure the lookup lasts until the future
+                  // finishes.
+                  return std::move(future).ensure(
+                      [proc = std::move(lookup)] {});
+                })
             .deferValue([parentTree = std::move(parentTree)](auto&&) mutable {
               return parentTree;
             });
@@ -1041,66 +1116,6 @@ std::shared_ptr<const Tree> EdenMount::getCheckedOutRootTree() const {
   return parentState_.rlock()->checkedOutRootTree;
 }
 
-namespace {
-
-class TreeLookupProcessor {
- public:
-  explicit TreeLookupProcessor(
-      RelativePathPiece path,
-      std::shared_ptr<ObjectStore> objectStore,
-      ObjectFetchContext& context)
-      : path_{path},
-        iterRange_{path_.components()},
-        iter_{iterRange_.begin()},
-        objectStore_{std::move(objectStore)},
-        context_{context} {}
-
-  ImmediateFuture<std::variant<std::shared_ptr<const Tree>, TreeEntry>> next(
-      std::shared_ptr<const Tree> tree) {
-    using RetType = std::variant<std::shared_ptr<const Tree>, TreeEntry>;
-    auto name = *iter_++;
-    auto it = tree->find(name);
-
-    if (it == tree->cend()) {
-      return makeImmediateFuture<RetType>(
-          std::system_error(ENOENT, std::generic_category()));
-    }
-
-    if (iter_ == iterRange_.end()) {
-      if (it->second.isTree()) {
-        return objectStore_->getTree(it->second.getHash(), context_)
-            .thenValue([](std::shared_ptr<const Tree> tree) -> RetType {
-              return tree;
-            });
-      } else {
-        return ImmediateFuture{RetType{it->second}};
-      }
-    } else {
-      if (!it->second.isTree()) {
-        return makeImmediateFuture<RetType>(
-            std::system_error(ENOTDIR, std::generic_category()));
-      } else {
-        return objectStore_->getTree(it->second.getHash(), context_)
-            .thenValue([this](std::shared_ptr<const Tree> tree) {
-              return next(std::move(tree));
-            });
-      }
-    }
-  }
-
- private:
-  RelativePath path_;
-  RelativePath::base_type::component_iterator_range iterRange_;
-  RelativePath::base_type::component_iterator iter_;
-  std::shared_ptr<ObjectStore> objectStore_;
-  // The ObjectFetchContext is allocated at the beginning of a request and
-  // released once the request completes. Since the lifetime of
-  // TreeLookupProcessor is strictly less than the one of a request, we can
-  // safely store a reference to the fetch context.
-  ObjectFetchContext& context_;
-};
-} // namespace
-
 ImmediateFuture<std::variant<std::shared_ptr<const Tree>, TreeEntry>>
 EdenMount::getTreeOrTreeEntry(
     RelativePathPiece path,

eden/fs/inodes/Overlay.cpp

@@ -145,7 +145,8 @@ struct statfs Overlay::statFs() {
 
 folly::SemiFuture<Unit> Overlay::initialize(
     std::optional<AbsolutePath> mountPath,
-    OverlayChecker::ProgressCallback&& progressCallback) {
+    OverlayChecker::ProgressCallback&& progressCallback,
+    OverlayChecker::LookupCallback&& lookupCallback) {
   // The initOverlay() call is potentially slow, so we want to avoid
   // performing it in the current thread and blocking returning to our caller.
   //
@@ -157,9 +158,11 @@ folly::SemiFuture<Unit> Overlay::initialize(
   gcThread_ = std::thread([this,
                            mountPath = std::move(mountPath),
                            progressCallback = std::move(progressCallback),
+                           lookupCallback = std::move(lookupCallback),
                            promise = std::move(initPromise)]() mutable {
     try {
-      initOverlay(std::move(mountPath), progressCallback);
+      initOverlay(
+          std::move(mountPath), progressCallback, std::move(lookupCallback));
     } catch (std::exception& ex) {
       XLOG(ERR) << "overlay initialization failed for "
                 << backingOverlay_->getLocalDir() << ": " << ex.what();
@@ -176,8 +179,8 @@ folly::SemiFuture<Unit> Overlay::initialize(
 
 void Overlay::initOverlay(
     std::optional<AbsolutePath> mountPath,
-    FOLLY_MAYBE_UNUSED const OverlayChecker::ProgressCallback&
-        progressCallback) {
+    FOLLY_MAYBE_UNUSED const OverlayChecker::ProgressCallback& progressCallback,
+    FOLLY_MAYBE_UNUSED OverlayChecker::LookupCallback&& lookupCallback) {
   IORequest req{this};
   auto optNextInodeNumber = backingOverlay_->initOverlay(true);
   if (!optNextInodeNumber.has_value()) {
@@ -195,7 +198,9 @@ void Overlay::initOverlay(
     // TODO(zeyi): `OverlayCheck` should be associated with the specific
     // Overlay implementation. `reinterpret_cast` is a temporary workaround.
     OverlayChecker checker(
-        reinterpret_cast<FsOverlay*>(backingOverlay_.get()), std::nullopt);
+        reinterpret_cast<FsOverlay*>(backingOverlay_.get()),
+        std::nullopt,
+        std::move(lookupCallback));
     folly::stop_watch<> fsckRuntime;
     checker.scanForErrors(progressCallback);
     auto result = checker.repairErrors();

eden/fs/inodes/Overlay.h

@@ -22,6 +22,7 @@
 #include "eden/fs/telemetry/StructuredLogger.h"
 #include "eden/fs/utils/CaseSensitivity.h"
 #include "eden/fs/utils/DirType.h"
+#include "eden/fs/utils/ImmediateFuture.h"
 #include "eden/fs/utils/PathFuncs.h"
 
 #ifndef _WIN32
@@ -114,7 +115,12 @@ class Overlay : public std::enable_shared_from_this<Overlay> {
    */
   FOLLY_NODISCARD folly::SemiFuture<folly::Unit> initialize(
       std::optional<AbsolutePath> mountPath = std::nullopt,
-      OverlayChecker::ProgressCallback&& progressCallback = [](auto) {});
+      OverlayChecker::ProgressCallback&& progressCallback = [](auto) {},
+      OverlayChecker::LookupCallback&& lookupCallback =
+          [](auto) {
+            return makeImmediateFuture<OverlayChecker::LookupCallbackValue>(
+                std::runtime_error("no lookup callback"));
+          });
 
   /**
    * Closes the overlay. It is undefined behavior to access the
@@ -297,7 +303,8 @@ class Overlay : public std::enable_shared_from_this<Overlay> {
 
   void initOverlay(
       std::optional<AbsolutePath> mountPath,
-      const OverlayChecker::ProgressCallback& progressCallback = [](auto) {});
+      const OverlayChecker::ProgressCallback& progressCallback,
+      OverlayChecker::LookupCallback&& lookupCallback);
   void gcThread() noexcept;
   void handleGCRequest(GCRequest& request);
 

eden/fs/inodes/fsoverlay/OverlayChecker.cpp

@@ -655,8 +655,11 @@ class OverlayChecker::BadNextInodeNumber : public OverlayChecker::Error {
 
 OverlayChecker::OverlayChecker(
     FsOverlay* fs,
-    optional<InodeNumber> nextInodeNumber)
-    : fs_(fs), loadedNextInodeNumber_(nextInodeNumber) {}
+    optional<InodeNumber> nextInodeNumber,
+    LookupCallback&& lookupCallback)
+    : fs_(fs),
+      loadedNextInodeNumber_(nextInodeNumber),
+      lookupCallback_(std::move(lookupCallback)) {}
 
 OverlayChecker::~OverlayChecker() {}
 
@@ -783,6 +786,11 @@ OverlayChecker::getInodeInfo(InodeNumber number) {
   return &(iter->second);
 }
 
+ImmediateFuture<std::variant<std::shared_ptr<const Tree>, TreeEntry>>
+OverlayChecker::lookup(RelativePathPiece path) {
+  return lookupCallback_(path);
+}
+
 OverlayChecker::PathInfo OverlayChecker::computePath(InodeNumber number) {
   return cachedPathComputation(number, [&]() {
     auto info = getInodeInfo(number);

eden/fs/inodes/fsoverlay/OverlayChecker.h

@@ -15,6 +15,8 @@
 
 #include "eden/fs/inodes/InodeNumber.h"
 #include "eden/fs/inodes/overlay/gen-cpp2/overlay_types.h"
+#include "eden/fs/model/Tree.h"
+#include "eden/fs/utils/ImmediateFuture.h"
 #include "eden/fs/utils/PathFuncs.h"
 
 namespace folly {
@@ -49,6 +51,12 @@ class OverlayChecker {
     uint32_t fixedErrors{0};
   };
 
+  using ProgressCallback = std::function<void(uint16_t)>;
+  using LookupCallbackValue =
+      std::variant<std::shared_ptr<const Tree>, TreeEntry>;
+  using LookupCallback =
+      std::function<ImmediateFuture<LookupCallbackValue>(RelativePathPiece)>;
+
   /**
    * Create a new OverlayChecker.
    *
@@ -56,12 +64,13 @@ class OverlayChecker {
    * of the check operation.  The caller is responsible for ensuring that the
    * FsOverlay object exists for at least as long as the OverlayChecker object.
    */
-  OverlayChecker(FsOverlay* fs, std::optional<InodeNumber> nextInodeNumber);
+  OverlayChecker(
+      FsOverlay* fs,
+      std::optional<InodeNumber> nextInodeNumber,
+      LookupCallback&& lookupCallback);
 
   ~OverlayChecker();
 
-  using ProgressCallback = std::function<void(uint16_t)>;
-
   /**
    * Scan the overlay for problems.
    */
@@ -180,6 +189,9 @@ class OverlayChecker {
   // readInodes() must have been called for inode info to be populated.
   InodeInfo* FOLLY_NULLABLE getInodeInfo(InodeNumber number);
 
+  ImmediateFuture<std::variant<std::shared_ptr<const Tree>, TreeEntry>> lookup(
+      RelativePathPiece path);
+
   PathInfo computePath(const InodeInfo& info);
   PathComponent findChildName(const InodeInfo& parentInfo, InodeNumber child);
   template <typename Fn>
@@ -216,6 +228,7 @@ class OverlayChecker {
 
   FsOverlay* const fs_;
   std::optional<InodeNumber> loadedNextInodeNumber_;
+  LookupCallback lookupCallback_;
   std::unordered_map<InodeNumber, InodeInfo> inodes_;
   std::vector<std::unique_ptr<Error>> errors_;
   uint64_t maxInodeNumber_{kRootNodeId.get()};

eden/fs/inodes/fsoverlay/eden_fsck.cpp

@@ -49,7 +49,10 @@ int main(int argc, char** argv) {
     XLOG(INFO) << "Overlay was shut down uncleanly";
   }
 
-  OverlayChecker checker(&fsOverlay.value(), nextInodeNumber);
+  OverlayChecker checker(&fsOverlay.value(), nextInodeNumber, [](auto&&) {
+    return makeImmediateFuture<OverlayChecker::LookupCallbackValue>(
+        std::runtime_error("no lookup callback"));
+  });
   checker.scanForErrors();
   if (FLAGS_dry_run) {
     checker.logErrors();

eden/fs/inodes/fsoverlay/test/FsckTest.cpp

@@ -327,7 +327,10 @@ TEST(Fsck, testNoErrors) {
 
   FsOverlay fs(overlay->overlayPath());
   auto nextInode = fs.initOverlay(/*createIfNonExisting=*/false);
-  OverlayChecker checker(&fs, nextInode);
+  OverlayChecker checker(&fs, nextInode, [](auto&&) {
+    return makeImmediateFuture<OverlayChecker::LookupCallbackValue>(
+        std::runtime_error("no lookup callback"));
+  });
   checker.scanForErrors();
   EXPECT_EQ(0, checker.getErrors().size());
   EXPECT_THAT(errorMessages(checker), UnorderedElementsAre());
@@ -360,7 +363,10 @@ TEST(Fsck, testMissingNextInodeNumber) {
   auto nextInode = fs.initOverlay(/*createIfNonExisting=*/false);
   // Confirm there is no next inode data
   EXPECT_FALSE(nextInode.has_value());
-  OverlayChecker checker(&fs, nextInode);
+  OverlayChecker checker(&fs, nextInode, [](auto&&) {
+    return makeImmediateFuture<OverlayChecker::LookupCallbackValue>(
+        std::runtime_error("no lookup callback"));
+  });
   checker.scanForErrors();
   // OverlayChecker should still report 0 errors in this case.
   // We don't report a missing next inode number as an error: if this is the
@@ -383,7 +389,10 @@ TEST(Fsck, testBadNextInodeNumber) {
   FsOverlay fs(overlay->overlayPath());
   auto nextInode = fs.initOverlay(/*createIfNonExisting=*/false);
   EXPECT_EQ(2, nextInode ? nextInode->get() : 0);
-  OverlayChecker checker(&fs, nextInode);
+  OverlayChecker checker(&fs, nextInode, [](auto&&) {
+    return makeImmediateFuture<OverlayChecker::LookupCallbackValue>(
+        std::runtime_error("no lookup callback"));
+  });
   checker.scanForErrors();
   EXPECT_THAT(
       errorMessages(checker),
@@ -403,7 +412,10 @@ TEST(Fsck, testBadFileData) {
   std::string badHeader(FsOverlay::kHeaderLength, 0x55);
   overlay->corruptInodeHeader(layout.src_foo_testTxt.number(), badHeader);
 
-  OverlayChecker checker(&overlay->fs(), std::nullopt);
+  OverlayChecker checker(&overlay->fs(), std::nullopt, [](auto&&) {
+    return makeImmediateFuture<OverlayChecker::LookupCallbackValue>(
+        std::runtime_error("no lookup callback"));
+  });
   checker.scanForErrors();
   EXPECT_THAT(
       errorMessages(checker),
@@ -444,7 +456,10 @@ TEST(Fsck, testTruncatedDirData) {
   auto srcDataFile = overlay->fs().openFileNoVerify(layout.src.number());
   folly::checkUnixError(ftruncate(srcDataFile.fd(), 0), "truncate failed");
 
-  OverlayChecker checker(&overlay->fs(), std::nullopt);
+  OverlayChecker checker(&overlay->fs(), std::nullopt, [](auto&&) {
+    return makeImmediateFuture<OverlayChecker::LookupCallbackValue>(
+        std::runtime_error("no lookup callback"));
+  });
   checker.scanForErrors();
   EXPECT_THAT(
       errorMessages(checker),
@@ -522,7 +537,10 @@ TEST(Fsck, testMissingDirData) {
   // subtree.
   overlay->fs().removeOverlayData(layout.src_foo_x.number());
 
-  OverlayChecker checker(&overlay->fs(), std::nullopt);
+  OverlayChecker checker(&overlay->fs(), std::nullopt, [](auto&&) {
+    return makeImmediateFuture<OverlayChecker::LookupCallbackValue>(
+        std::runtime_error("no lookup callback"));
+  });
   checker.scanForErrors();
   EXPECT_THAT(
       errorMessages(checker),
@@ -599,7 +617,10 @@ TEST(Fsck, testHardLink) {
   layout.src_foo.linkFile(layout.src_foo_x_y_zTxt.number(), "also_z.txt");
   layout.src_foo.save();
 
-  OverlayChecker checker(&overlay->fs(), std::nullopt);
+  OverlayChecker checker(&overlay->fs(), std::nullopt, [](auto&&) {
+    return makeImmediateFuture<OverlayChecker::LookupCallbackValue>(
+        std::runtime_error("no lookup callback"));
+  });
   checker.scanForErrors();
   EXPECT_THAT(
       errorMessages(checker),