Remove HgImporter support for fetching trees

Summary: HgImporter is being removed - this change removes its support for fetching trees.

Reviewed By: kmancini

Differential Revision: D51872978

fbshipit-source-id: 5b373565c6b29ecc3dc7988695b6e4f62d9a04fb

eden/fs/store/hg/HgBackingStore.cpp

@@ -75,16 +75,6 @@ namespace {
 // Thread local HgImporter. This is only initialized on HgImporter threads.
 static folly::ThreadLocalPtr<Importer> threadLocalImporter;
 
-/**
- * Checks that the thread local HgImporter is present and returns it.
- */
-Importer& getThreadLocalImporter() {
-  if (!threadLocalImporter) {
-    EDEN_BUG() << "Attempting to get HgImporter from non-HgImporter thread";
-  }
-  return *threadLocalImporter;
-}
-
 ObjectId hashFromRootId(const RootId& root) {
   return ObjectId::fromHex(root.value());
 }
@@ -313,7 +303,6 @@ folly::Future<TreePtr> HgBackingStore::fetchTreeFromImporter(
               edenTreeID,
               writeBatch,
               &liveImportTreeWatches = liveImportTreeWatches_] {
-               Importer& importer = getThreadLocalImporter();
                folly::stop_watch<std::chrono::milliseconds> watch;
                RequestMetricsScope queueTracker{&liveImportTreeWatches};
 
@@ -335,23 +324,8 @@ folly::Future<TreePtr> HgBackingStore::fetchTreeFromImporter(
                if (tree.hasValue()) {
                  stats_->increment(&HgBackingStoreStats::fetchTreeRetrySuccess);
                  result = tree.value();
-               } else if (config_->getEdenConfig()
-                              ->hgImporterFetchFallback.getValue()) {
-                 // Fall back to importer
-                 auto serializedTree = importer.fetchTree(path, manifestNode);
-                 if (serializedTree) {
-                   stats_->increment(&HgBackingStoreStats::importTreeSuccess);
-                 } else {
-                   stats_->increment(&HgBackingStoreStats::importTreeFailure);
-                 }
-                 result = processTree(
-                     std::move(serializedTree),
-                     manifestNode,
-                     edenTreeID,
-                     path,
-                     writeBatch.get());
                } else {
-                 // No fallback to importer, record miss and return error
+                 // Record miss and return error
                  if (logger_) {
                    logger_->logEvent(FetchMiss{
                        datapackStore_.getRepoName(),
@@ -484,37 +458,6 @@ class Manifest {
 
 } // namespace
 
-TreePtr HgBackingStore::processTree(
-    std::unique_ptr<IOBuf> content,
-    const Hash20& manifestNode,
-    const ObjectId& edenTreeID,
-    RelativePathPiece path,
-    LocalStore::WriteBatch* writeBatch) {
-  auto manifest = Manifest(std::move(content));
-  Tree::container entries{kPathMapDefaultCaseSensitive};
-  auto hgObjectIdFormat = config_->getEdenConfig()->hgObjectIdFormat.getValue();
-  const auto filteredPaths =
-      config_->getEdenConfig()->hgFilteredPaths.getValue();
-
-  for (auto& entry : manifest) {
-    XLOG(DBG9) << "tree: " << manifestNode << " " << entry.name
-               << " node: " << entry.node << " flag: " << entry.type;
-
-    auto relPath = path + entry.name;
-    if (filteredPaths->count(relPath) == 0) {
-      auto proxyHash =
-          HgProxyHash::store(relPath, entry.node, hgObjectIdFormat);
-
-      entries.emplace(entry.name, proxyHash, entry.type);
-    }
-  }
-
-  writeBatch->flush();
-
-  return std::make_shared<TreePtr::element_type>(
-      std::move(entries), edenTreeID);
-}
-
 ImmediateFuture<folly::Unit> HgBackingStore::importTreeManifestForRoot(
     const RootId& rootId,
     const Hash20& manifestId,

eden/fs/store/hg/HgBackingStore.h

@@ -161,12 +161,6 @@ class HgBackingStore {
       ObjectId edenTreeID,
       RelativePath path,
       std::shared_ptr<LocalStore::WriteBatch> writeBatch);
-  TreePtr processTree(
-      std::unique_ptr<folly::IOBuf> content,
-      const Hash20& manifestNode,
-      const ObjectId& edenTreeID,
-      RelativePathPiece path,
-      LocalStore::WriteBatch* writeBatch);
 
   std::shared_ptr<LocalStore> localStore_;
   EdenStatsPtr stats_;

eden/fs/store/hg/HgImporter.cpp

@@ -255,64 +255,6 @@ void HgImporter::stopHelperProcess() {
   }
 }
 
-std::unique_ptr<IOBuf> HgImporter::fetchTree(
-    RelativePathPiece path,
-    Hash20 pathManifestNode) {
-  // Ask the hg_import_helper script to fetch data for this tree
-  static constexpr auto getNumRequestsSinceLastLog =
-      [](uint64_t& treeRequestsSinceLog) {
-        uint64_t numRequests = 0;
-        std::swap(numRequests, treeRequestsSinceLog);
-        return numRequests;
-      };
-  XLOG_EVERY_MS(DBG1, 1000)
-      << "fetching data for tree \"" << path << "\" at manifest node "
-      << pathManifestNode << ". "
-      << getNumRequestsSinceLastLog(treeRequestsSinceLog_)
-      << " trees fetched since last log";
-  treeRequestsSinceLog_++;
-
-  auto requestID = sendFetchTreeRequest(
-      CMD_CAT_TREE, path, pathManifestNode, "CMD_CAT_TREE");
-
-  ChunkHeader header;
-  header = readChunkHeader(requestID, "CMD_CAT_TREE");
-
-  auto buf = IOBuf::create(header.dataLength);
-
-  readFromHelper(
-      buf->writableTail(), header.dataLength, "CMD_CAT_TREE response body");
-  buf->append(header.dataLength);
-
-  // The last 8 bytes of the response are the body length.
-  // Ensure that this looks correct, and advance the buffer past this data to
-  // the start of the actual response body.
-  //
-  // This data doesn't really need to be present in the response.  It is only
-  // here so we can double-check that the response data appears valid.
-  buf->trimEnd(sizeof(uint64_t));
-  uint64_t bodyLength;
-  memcpy(&bodyLength, buf->tail(), sizeof(uint64_t));
-  bodyLength = Endian::big(bodyLength);
-  if (bodyLength != header.dataLength - sizeof(uint64_t)) {
-    auto msg = fmt::format(
-        "inconsistent body length received when importing tree {} ({}, {}): "
-        "bodyLength={} responseLength={}",
-        pathManifestNode,
-        path,
-        pathManifestNode,
-        bodyLength,
-        header.dataLength);
-    XLOG(ERR) << msg;
-    throw std::runtime_error(std::move(msg));
-  }
-
-  XLOG(DBG4) << "imported tree " << pathManifestNode << " (" << path << ", "
-             << pathManifestNode << "); length=" << bodyLength;
-
-  return buf;
-}
-
 HgImporter::ChunkHeader HgImporter::readChunkHeader(
     TransactionID txnID,
     StringPiece cmdName) {
@@ -362,34 +304,6 @@ HgImporter::ChunkHeader HgImporter::readChunkHeader(
   throw HgImportPyError(errorType, message);
 }
 
-HgImporter::TransactionID HgImporter::sendFetchTreeRequest(
-    CommandType cmd,
-    RelativePathPiece path,
-    Hash20 pathManifestNode,
-    StringPiece context) {
-  stats_->increment(&HgImporterStats::fetchTree);
-
-  auto txnID = nextRequestID_++;
-  ChunkHeader header;
-  header.command = Endian::big<uint32_t>(cmd);
-  header.requestID = Endian::big<uint32_t>(txnID);
-  header.flags = 0;
-  std::string_view pathStr = path.view();
-  header.dataLength = Endian::big<uint32_t>(
-      folly::to_narrow(Hash20::RAW_SIZE + pathStr.size()));
-
-  std::array<struct iovec, 3> iov;
-  iov[0].iov_base = &header;
-  iov[0].iov_len = sizeof(header);
-  iov[1].iov_base = const_cast<uint8_t*>(pathManifestNode.getBytes().data());
-  iov[1].iov_len = Hash20::RAW_SIZE;
-  iov[2].iov_base = const_cast<char*>(pathStr.data());
-  iov[2].iov_len = pathStr.size();
-  writeToHelper(iov, context);
-
-  return txnID;
-}
-
 void HgImporter::readFromHelper(void* buf, uint32_t size, StringPiece context) {
   size_t bytesRead;
 
@@ -490,16 +404,6 @@ auto HgImporterManager::retryOnError(Fn&& fn, FetchMiss::MissType missType) {
   }
 }
 
-std::unique_ptr<IOBuf> HgImporterManager::fetchTree(
-    RelativePathPiece path,
-    Hash20 pathManifestNode) {
-  return retryOnError(
-      [&](HgImporter* importer) {
-        return importer->fetchTree(path, pathManifestNode);
-      },
-      FetchMiss::Tree);
-}
-
 HgImporter* HgImporterManager::getImporter() {
   if (!importer_) {
     importer_ =

eden/fs/store/hg/HgImporter.h

@@ -57,16 +57,6 @@ struct ImporterOptions {
 class Importer {
  public:
   virtual ~Importer() = default;
-
-  /**
-   * Import tree and store it in the datapack
-   *
-   * In the case where CMD_CAT_TREE is used, a valid IOBuf is return,
-   * otherwise, nullptr is returned.
-   */
-  virtual std::unique_ptr<folly::IOBuf> fetchTree(
-      RelativePathPiece path,
-      Hash20 pathManifestNode) = 0;
 };
 
 /**
@@ -101,10 +91,6 @@ class HgImporter : public Importer {
 
   ProcessStatus debugStopHelperProcess();
 
-  std::unique_ptr<folly::IOBuf> fetchTree(
-      RelativePathPiece path,
-      Hash20 pathManifestNode) override;
-
   const ImporterOptions& getOptions() const;
 
  private:
@@ -204,24 +190,9 @@ class HgImporter : public Importer {
     writeToHelper(iov.data(), iov.size(), context);
   }
 
-  /**
-   * Send a request to the helper process asking it to prefetch data for trees
-   * under the specified path, at the specified manifest node for the given
-   * path.
-   */
-  TransactionID sendFetchTreeRequest(
-      CommandType cmd,
-      RelativePathPiece path,
-      Hash20 pathManifestNode,
-      folly::StringPiece context);
-
   SpawnedProcess helper_;
   EdenStatsPtr const stats_;
   ImporterOptions options_;
-  uint32_t nextRequestID_{0};
-
-  // How many trees were fetched since the last time we logged?
-  uint64_t treeRequestsSinceLog_{0};
 
   /**
    * The input and output file descriptors to the helper subprocess.
@@ -256,10 +227,6 @@ class HgImporterManager : public Importer {
       std::shared_ptr<StructuredLogger> logger,
       std::optional<AbsolutePath> importHelperScript = std::nullopt);
 
-  std::unique_ptr<folly::IOBuf> fetchTree(
-      RelativePathPiece path,
-      Hash20 pathManifestNode) override;
-
  private:
   template <typename Fn>
   auto retryOnError(Fn&& fn, FetchMiss::MissType missType);