Log files accumulated to ODS

Summary: Keep track of the longest query we had to resolve in accumulateRange. This stat will be helpful in determining how large the memory limit of the journal should be since we will know how far back people usually need to go.

Reviewed By: strager

Differential Revision: D16227920

fbshipit-source-id: a41c3b9f16b701cd8165e20409888983b8899dab

eden/fs/inodes/EdenMount.cpp

@@ -909,6 +909,8 @@ std::string EdenMount::getCounterName(CounterName name) {
       return "journal." + base + ".count";
     case CounterName::JOURNAL_DURATION:
       return "journal." + base + ".duration_secs";
+    case CounterName::JOURNAL_MAX_FILES_ACCUMULATED:
+      return "journal." + base + ".files_accumulated.max";
   }
   EDEN_BUG() << "unknown counter name " << static_cast<int>(name);
   folly::assume_unreachable();

eden/fs/inodes/EdenMount.h

@@ -92,6 +92,10 @@ enum class CounterName {
    * Represents the duration of the journal in seconds end to end
    */
   JOURNAL_DURATION,
+  /**
+   * Represents the maximum deltas iterated over in the Journal's forEachDelta
+   */
+  JOURNAL_MAX_FILES_ACCUMULATED
 };
 
 /**

eden/fs/journal/Journal.cpp

@@ -208,16 +208,17 @@ size_t Journal::getMemoryLimit() const {
   return deltaState->memoryLimit;
 }
 
-std::unique_ptr<JournalDeltaRange> Journal::accumulateRange() const {
+std::unique_ptr<JournalDeltaRange> Journal::accumulateRange() {
   return accumulateRange(1);
 }
 
 std::unique_ptr<JournalDeltaRange> Journal::accumulateRange(
-    SequenceNumber from) const {
+    SequenceNumber from) {
   DCHECK(from > 0);
   std::unique_ptr<JournalDeltaRange> result = nullptr;
 
-  auto deltaState = deltaState_.rlock();
+  size_t filesAccumulated = 0;
+  auto deltaState = deltaState_.ulock();
   // If this is going to be truncated handle it before iterating.
   if (!deltaState->deltas.empty() &&
       deltaState->deltas.front().sequenceID > from) {
@@ -228,7 +229,8 @@ std::unique_ptr<JournalDeltaRange> Journal::accumulateRange(
         deltaState->deltas,
         from,
         std::nullopt,
-        [&result](const JournalDelta& current) -> void {
+        [&result, &filesAccumulated](const JournalDelta& current) -> void {
+          ++filesAccumulated;
           if (!result) {
             result = std::make_unique<JournalDeltaRange>();
             result->toSequence = current.sequenceID;
@@ -265,9 +267,19 @@ std::unique_ptr<JournalDeltaRange> Journal::accumulateRange(
         });
   }
 
-  if (result && result->isTruncated) {
+  if (result) {
     if (edenStats_) {
-      edenStats_->getJournalStatsForCurrentThread().truncatedReads.addValue(1);
+      if (result->isTruncated) {
+        edenStats_->getJournalStatsForCurrentThread().truncatedReads.addValue(
+            1);
+      }
+      edenStats_->getJournalStatsForCurrentThread().filesAccumulated.addValue(
+          filesAccumulated);
+    }
+    auto deltaStateWriter = deltaState.moveFromUpgradeToWrite();
+    if (deltaStateWriter->stats) {
+      deltaStateWriter->stats->maxFilesAccumulated = std::max(
+          deltaStateWriter->stats->maxFilesAccumulated, filesAccumulated);
     }
   }
 

eden/fs/journal/Journal.h

@@ -27,6 +27,7 @@ struct JournalStats {
   size_t memoryUsage = 0;
   std::chrono::steady_clock::time_point earliestTimestamp;
   std::chrono::steady_clock::time_point latestTimestamp;
+  size_t maxFilesAccumulated = 0;
   uint64_t getDurationInSeconds() {
     return std::chrono::duration_cast<std::chrono::seconds>(
                std::chrono::steady_clock::now() - earliestTimestamp)
@@ -140,8 +141,8 @@ class Journal {
    * If the limitSequence means that no deltas will match, returns nullptr.
    * */
   std::unique_ptr<JournalDeltaRange> accumulateRange(
-      SequenceNumber limitSequence) const;
-  std::unique_ptr<JournalDeltaRange> accumulateRange() const;
+      SequenceNumber limitSequence);
+  std::unique_ptr<JournalDeltaRange> accumulateRange();
 
   /** Gets a vector of the modifications (newer deltas having lower indices)
    * done by the latest 'limit' deltas, if the

eden/fs/journal/test/JournalTest.cpp

@@ -240,6 +240,35 @@ TEST(Journal, truncated_read_stats) {
       2, edenStats->getJournalStatsForCurrentThread().truncatedReads.sum());
 }
 
+TEST(Journal, files_accumulated_stats) {
+  // Since each test is run on a single thread we can check that the stats of
+  // this thread match up with what we would expect.
+  auto edenStats = std::make_shared<EdenStats>();
+  Journal journal(edenStats);
+  journal.recordCreated("test1.txt"_relpath);
+  journal.recordRemoved("test1.txt"_relpath);
+  ASSERT_EQ(
+      0, edenStats->getJournalStatsForCurrentThread().filesAccumulated.sum());
+  ASSERT_EQ(0, journal.getStats()->maxFilesAccumulated);
+  // Empty Accumulate range, should be 0 files accumulated
+  journal.accumulateRange(3);
+  ASSERT_EQ(
+      0, edenStats->getJournalStatsForCurrentThread().filesAccumulated.sum());
+  ASSERT_EQ(0, journal.getStats()->maxFilesAccumulated);
+  journal.accumulateRange(2);
+  ASSERT_EQ(
+      1, edenStats->getJournalStatsForCurrentThread().filesAccumulated.sum());
+  ASSERT_EQ(1, journal.getStats()->maxFilesAccumulated);
+  journal.accumulateRange(1);
+  ASSERT_EQ(
+      3, edenStats->getJournalStatsForCurrentThread().filesAccumulated.sum());
+  ASSERT_EQ(2, journal.getStats()->maxFilesAccumulated);
+  journal.accumulateRange(2);
+  ASSERT_EQ(
+      4, edenStats->getJournalStatsForCurrentThread().filesAccumulated.sum());
+  ASSERT_EQ(2, journal.getStats()->maxFilesAccumulated);
+}
+
 TEST(Journal, memory_usage) {
   Journal journal(std::make_shared<EdenStats>());
   auto stats = journal.getStats();

eden/fs/service/EdenServer.cpp

@@ -778,6 +778,12 @@ void EdenServer::registerStats(std::shared_ptr<EdenMount> edenMount) {
         auto stats = edenMount->getJournal().getStats();
         return stats ? stats->getDurationInSeconds() : 0;
       });
+  counters->registerCallback(
+      edenMount->getCounterName(CounterName::JOURNAL_MAX_FILES_ACCUMULATED),
+      [edenMount] {
+        auto stats = edenMount->getJournal().getStats();
+        return stats ? stats->maxFilesAccumulated : 0;
+      });
 #else
   NOT_IMPLEMENTED();
 #endif // !_WIN32
@@ -795,6 +801,8 @@ void EdenServer::unregisterStats(EdenMount* edenMount) {
       edenMount->getCounterName(CounterName::JOURNAL_ENTRIES));
   counters->unregisterCallback(
       edenMount->getCounterName(CounterName::JOURNAL_DURATION));
+  counters->unregisterCallback(
+      edenMount->getCounterName(CounterName::JOURNAL_MAX_FILES_ACCUMULATED));
 #else
   NOT_IMPLEMENTED();
 #endif // !_WIN32

eden/fs/tracing/EdenStats.h

@@ -213,6 +213,10 @@ class HgImporterThreadStats : public EdenThreadStatsBase {
 class JournalThreadStats : public EdenThreadStatsBase {
  public:
   Timeseries truncatedReads{this, "journal.truncated_reads", fb303::SUM};
+  Timeseries filesAccumulated{this,
+                              "journal.files_accumulated",
+                              fb303::COUNT,
+                              fb303::SUM};
 };
 
 } // namespace eden