Adding truncatedReads stat to journal

Summary: The journal will now keep of how many reads from accumulateRange have been truncated. This is a useful metric that will allow us to see how often we are forcing Watchman to use its fallback of creating a fresh instance and help us in calibrating if we find our memory limit is too small.

Reviewed By: strager

Differential Revision: D16017968

fbshipit-source-id: 95f4fbd1fd2d8523ff397202172408e1c89669be

eden/fs/journal/Journal.cpp

@@ -216,53 +216,61 @@ std::unique_ptr<JournalDeltaRange> Journal::accumulateRange(
     SequenceNumber from) const {
   DCHECK(from > 0);
   std::unique_ptr<JournalDeltaRange> result = nullptr;
+
   auto deltaState = deltaState_.rlock();
   // If this is going to be truncated handle it before iterating.
   if (!deltaState->deltas.empty() &&
       deltaState->deltas.front().sequenceID > from) {
     result = std::make_unique<JournalDeltaRange>();
     result->isTruncated = true;
-    return result;
-  }
-  forEachDelta(
-      deltaState->deltas,
-      from,
-      std::nullopt,
-      [&result](const JournalDelta& current) -> void {
-        if (!result) {
-          result = std::make_unique<JournalDeltaRange>();
-          result->toSequence = current.sequenceID;
-          result->toTime = current.time;
-          result->toHash = current.toHash;
-        }
-        // Capture the lower bound.
-        result->fromSequence = current.sequenceID;
-        result->fromTime = current.time;
-        result->fromHash = current.fromHash;
-
-        // Merge the unclean status list
-        result->uncleanPaths.insert(
-            current.uncleanPaths.begin(), current.uncleanPaths.end());
-
-        for (auto& entry : current.getChangedFilesInOverlay()) {
-          auto& name = entry.first;
-          auto& currentInfo = entry.second;
-          auto* resultInfo =
-              folly::get_ptr(result->changedFilesInOverlay, name);
-          if (!resultInfo) {
-            result->changedFilesInOverlay.emplace(name, currentInfo);
-          } else {
-            if (resultInfo->existedBefore != currentInfo.existedAfter) {
-              auto event1 = eventCharacterizationFor(currentInfo);
-              auto event2 = eventCharacterizationFor(*resultInfo);
-              XLOG(ERR) << "Journal for " << name << " holds invalid " << event1
-                        << ", " << event2 << " sequence";
-            }
-
-            resultInfo->existedBefore = currentInfo.existedBefore;
+  } else {
+    forEachDelta(
+        deltaState->deltas,
+        from,
+        std::nullopt,
+        [&result](const JournalDelta& current) -> void {
+          if (!result) {
+            result = std::make_unique<JournalDeltaRange>();
+            result->toSequence = current.sequenceID;
+            result->toTime = current.time;
+            result->toHash = current.toHash;
           }
-        }
-      });
+          // Capture the lower bound.
+          result->fromSequence = current.sequenceID;
+          result->fromTime = current.time;
+          result->fromHash = current.fromHash;
+
+          // Merge the unclean status list
+          result->uncleanPaths.insert(
+              current.uncleanPaths.begin(), current.uncleanPaths.end());
+
+          for (auto& entry : current.getChangedFilesInOverlay()) {
+            auto& name = entry.first;
+            auto& currentInfo = entry.second;
+            auto* resultInfo =
+                folly::get_ptr(result->changedFilesInOverlay, name);
+            if (!resultInfo) {
+              result->changedFilesInOverlay.emplace(name, currentInfo);
+            } else {
+              if (resultInfo->existedBefore != currentInfo.existedAfter) {
+                auto event1 = eventCharacterizationFor(currentInfo);
+                auto event2 = eventCharacterizationFor(*resultInfo);
+                XLOG(ERR) << "Journal for " << name << " holds invalid "
+                          << event1 << ", " << event2 << " sequence";
+              }
+
+              resultInfo->existedBefore = currentInfo.existedBefore;
+            }
+          }
+        });
+  }
+
+  if (result && result->isTruncated) {
+    if (edenStats_) {
+      edenStats_->getJournalStatsForCurrentThread().truncatedReads.addValue(1);
+    }
+  }
+
   return result;
 }
 

eden/fs/journal/Journal.h

@@ -60,7 +60,10 @@ struct JournalDeltaInfo {
 class Journal {
  public:
   explicit Journal(std::shared_ptr<EdenStats> edenStats)
-      : edenStats_{std::move(edenStats)} {}
+      : edenStats_{std::move(edenStats)} {
+    /** Add 0 so that this counter shows up in ODS */
+    edenStats_->getJournalStatsForCurrentThread().truncatedReads.addValue(0);
+  }
 
   /// It is almost always a mistake to copy a Journal.
   Journal(const Journal&) = delete;

eden/fs/journal/test/JournalTest.cpp

@@ -211,6 +211,35 @@ TEST(Journal, basic_journal_stats) {
   ASSERT_EQ(to2, stats->latestTimestamp);
 }
 
+TEST(Journal, truncated_read_stats) {
+  // Since each test is run on a single thread we can check that the stats of
+  // this thread match up with what we would expect.
+  auto edenStats = std::make_shared<EdenStats>();
+  Journal journal(edenStats);
+  journal.setMemoryLimit(0);
+  journal.recordCreated("test1.txt"_relpath);
+  journal.recordRemoved("test1.txt"_relpath);
+  ASSERT_EQ(
+      0, edenStats->getJournalStatsForCurrentThread().truncatedReads.sum());
+  // Empty Accumulate range, should be 0 files accumulated
+  journal.accumulateRange(3);
+  ASSERT_EQ(
+      0, edenStats->getJournalStatsForCurrentThread().truncatedReads.sum());
+  // This is not a truncated read since journal remembers at least one entry
+  journal.accumulateRange(2);
+  ASSERT_EQ(
+      0, edenStats->getJournalStatsForCurrentThread().truncatedReads.sum());
+  journal.accumulateRange(1);
+  ASSERT_EQ(
+      1, edenStats->getJournalStatsForCurrentThread().truncatedReads.sum());
+  journal.accumulateRange(2);
+  ASSERT_EQ(
+      1, edenStats->getJournalStatsForCurrentThread().truncatedReads.sum());
+  journal.accumulateRange(1);
+  ASSERT_EQ(
+      2, edenStats->getJournalStatsForCurrentThread().truncatedReads.sum());
+}
+
 TEST(Journal, memory_usage) {
   Journal journal(std::make_shared<EdenStats>());
   auto stats = journal.getStats();

eden/fs/tracing/EdenStats.h

@@ -210,7 +210,10 @@ class HgImporterThreadStats : public EdenThreadStatsBase {
   Timeseries prefetchFiles{createTimeseries("hg_importer.prefetch_files")};
 };
 
-class JournalThreadStats : public EdenThreadStatsBase {};
+class JournalThreadStats : public EdenThreadStatsBase {
+ public:
+  Timeseries truncatedReads{this, "journal.truncated_reads", fb303::SUM};
+};
 
 } // namespace eden
 } // namespace facebook