ObjectStore stats

Summary: Added the cli command `eden stats object-store` for querying the counts on what part of the object store was responsible for finding the blob or blob size (local store or backing store). This will tell us how well local and in-memory caching works for different workflows.

Reviewed By: chadaustin

Differential Revision: D15934535

fbshipit-source-id: 70345f11a51c3c6996dc001d4101744395a3d182

eden/cli/stats.py

@@ -120,15 +120,15 @@ class MemoryCmd(Subcmd):
         instance = cmd_util.get_eden_instance(args)
 
         with instance.get_thrift_client() as client:
-            diag_info = client.getStatInfo()
-            stats_print.write_mem_status_table(diag_info.counters, out)
+            counters = client.getCounters()
+            stats_print.write_mem_status_table(counters, out)
 
             # print memory counters
             heading = "Average values of Memory usage and availability"
             out.write("\n\n %s \n\n" % heading.center(80, " "))
 
-            mem_counters = get_memory_counters(diag_info.counters)
-            stats_print.write_table(mem_counters, "", out)
+            table = get_memory_counters(counters)
+            stats_print.write_table(table, "", out)
 
         return 0
 
@@ -166,11 +166,11 @@ class IoCmd(Subcmd):
         stats_print.write_heading("Counts of I/O operations performed in EdenFs", out)
         instance = cmd_util.get_eden_instance(args)
         with instance.get_thrift_client() as client:
-            diag_info = client.getStatInfo()
+            counters = client.getCounters()
 
         # If the arguments has --all flag, we will have args.all set to
         # true.
-        fuse_counters = get_fuse_counters(diag_info.counters, args.all)
+        fuse_counters = get_fuse_counters(counters, args.all)
         stats_print.write_table(fuse_counters, "SystemCall", out)
 
         return 0
@@ -257,16 +257,15 @@ class LatencyCmd(Subcmd):
         )
 
     def run(self, args: argparse.Namespace) -> int:
-        out = sys.stdout
+        TITLE = "Latencies of I/O operations performed in EdenFS"
+        stats_print.write_heading(TITLE, sys.stdout)
+
         instance = cmd_util.get_eden_instance(args)
         with instance.get_thrift_client() as client:
-            diag_info = client.getStatInfo()
+            counters = client.getCounters()
 
-        table = get_fuse_latency(diag_info.counters, args.all)
-        stats_print.write_heading(
-            "Latencies of I/O operations performed in EdenFs", out
-        )
-        stats_print.write_latency_table(table, out)
+        table = get_fuse_latency(counters, args.all)
+        stats_print.write_latency_table(table, sys.stdout)
 
         return 0
 
@@ -319,95 +318,49 @@ def get_fuse_latency(counters: DiagInfoCounters, all_flg: bool) -> Table2D:
 )
 class HgImporterCmd(Subcmd):
     def run(self, args: argparse.Namespace) -> int:
-        out = sys.stdout
-        stats_print.write_heading(
-            "Counts of HgImporter requests performed in EdenFS", out
-        )
+        TITLE = "Counts of HgImporter requests performed in EdenFS"
+        stats_print.write_heading(TITLE, sys.stdout)
+
         instance = cmd_util.get_eden_instance(args)
         with instance.get_thrift_client() as client:
             counters = client.getCounters()
 
-        hg_importer_counters = get_hg_importer_counters(counters)
-        stats_print.write_table(hg_importer_counters, "HgImporter Request", out)
+        table = get_counter_table(counters, ["hg_importer"], ["count"])
+        stats_print.write_table(table, "HgImporter Request", sys.stdout)
 
         return 0
 
 
-def get_hg_importer_counters(counters: DiagInfoCounters) -> Table:
-    zero = [0, 0, 0, 0]
-    table: Table = {
-        "cat_file": zero,
-        "fetch_tree": zero,
-        "manifest": zero,
-        "manifest_node_for_commit": zero,
-        "prefetch_files": zero,
-    }
-
-    for key in counters:
-        segments = key.split(".")
-        if (
-            len(segments) == 3
-            and segments[0] == "hg_importer"
-            and segments[2] == "count"
-        ):
-            call_name = segments[1]
-            last_minute = counters[key + ".60"]
-            last_10_minutes = counters[key + ".600"]
-            last_hour = counters[key + ".3600"]
-            all_time = counters[key]
-            table[call_name] = [last_minute, last_10_minutes, last_hour, all_time]
-
-    return table
-
-
 @stats_cmd("thrift", "Show the number of received thrift calls")
 class ThriftCmd(Subcmd):
     def run(self, args: argparse.Namespace) -> int:
-        out = sys.stdout
-        stats_print.write_heading("Counts of Thrift calls performed in EdenFs", out)
+        TITLE = "Counts of Thrift calls performed in EdenFS"
+        stats_print.write_heading(TITLE, sys.stdout)
+
         instance = cmd_util.get_eden_instance(args)
         with instance.get_thrift_client() as client:
-            counters = client.getCounters()
+            counters = client.getRegexCounters("thrift.EdenService\\..*")
 
-        thrift_counters = get_thrift_counters(counters)
-        stats_print.write_table(thrift_counters, "Thrift Call", out)
+        PREFIX = ["thrift", "EdenService"]
+        SUFFIX = ["num_calls", "sum"]
+        table = get_counter_table(counters, PREFIX, SUFFIX)
+        stats_print.write_table(table, "Thrift Call", sys.stdout)
 
         return 0
 
 
-def get_thrift_counters(counters: DiagInfoCounters) -> Table:
-    table: Table = {}
-
-    for key in counters:
-        segments = key.split(".")
-        if (
-            len(segments) == 5
-            and segments[:2] == ["thrift", "EdenService"]
-            and segments[-2:] == ["num_calls", "sum"]
-        ):
-            call_name = segments[2]
-            last_minute = counters[key + ".60"]
-            last_10_minutes = counters[key + ".600"]
-            last_hour = counters[key + ".3600"]
-            all_time = counters[key]
-            table[call_name] = [last_minute, last_10_minutes, last_hour, all_time]
-
-    return table
-
-
 @stats_cmd("thrift-latency", "Show the latency of received thrift calls")
 class ThriftLatencyCmd(Subcmd):
     def run(self, args: argparse.Namespace) -> int:
-        out = sys.stdout
+        TITLE = "Latency of Thrift processing time performed in EdenFS"
+        stats_print.write_heading(TITLE, sys.stdout)
+
         instance = cmd_util.get_eden_instance(args)
         with instance.get_thrift_client() as client:
-            diag_info = client.getStatInfo()
+            counters = client.getCounters()
 
-        table = get_thrift_latency(diag_info.counters)
-        stats_print.write_heading(
-            "Latency of Thrift processing time performed in EdenFs", out
-        )
-        stats_print.write_latency_table(table, out)
+        table = get_thrift_latency(counters)
+        stats_print.write_latency_table(table, sys.stdout)
 
         return 0
 
@@ -441,15 +394,16 @@ class MononokeBackingStoreLatencyCmd(Subcmd):
 
 
 def backing_store_latency(store: str, args: argparse.Namespace) -> int:
-    out = sys.stdout
+    TITLE = "Latency of {} backing store operations in EdenFs".format(store)
+    stats_print.write_heading(TITLE, sys.stdout)
+
     instance = cmd_util.get_eden_instance(args)
     with instance.get_thrift_client() as client:
-        diag_info = client.getStatInfo()
-    table = get_store_latency(diag_info.counters, store)
-    stats_print.write_heading(
-        "Latency of {} backing store operations in EdenFs".format(store), out
-    )
-    stats_print.write_latency_table(table, out)
+        counters = client.getCounters()
+
+    table = get_store_latency(counters, store)
+    stats_print.write_latency_table(table, sys.stdout)
+
     return 0
 
 
@@ -468,7 +422,7 @@ def get_store_latency(counters: DiagInfoCounters, store: str) -> Table2D:
     return table
 
 
-@stats_cmd("local_store", "Show information about the local store data size")
+@stats_cmd("local-store", "Show information about the local store data size")
 class LocalStoreCmd(Subcmd):
     def run(self, args: argparse.Namespace) -> int:
         # (name, ephemeral)
@@ -482,6 +436,7 @@ class LocalStoreCmd(Subcmd):
 
         out = sys.stdout
         stats_print.write_heading("EdenFS Local Store Stats", out)
+
         instance = cmd_util.get_eden_instance(args)
         with instance.get_thrift_client() as client:
             counters = client.getRegexCounters("local_store\\..*")
@@ -525,6 +480,35 @@ class LocalStoreCmd(Subcmd):
         return 0
 
 
+@stats_cmd("object-store", "Show information about the object store pipeline")
+class ObjectStoreCommand(Subcmd):
+    def run(self, args: argparse.Namespace) -> int:
+        TITLE = "Percentages of where data was found by the object store"
+        stats_print.write_heading(TITLE, sys.stdout)
+
+        eden = cmd_util.get_eden_instance(args)
+        with eden.get_thrift_client() as thrift:
+            counters = thrift.getRegexCounters("object_store\\..*")
+
+        table = get_counter_table(counters, ["object_store"], ["pct"])
+        stats_print.write_table(table, "Object Store", sys.stdout)
+
+        return 0
+
+
+def get_counter_table(counters: DiagInfoCounters, prefix: List, suffix: List) -> Table:
+    table: Table = {}
+
+    for key in counters:
+        tags = key.split(".")
+        if tags[-len(suffix) :] == suffix:
+            TIME_SUFFIXES = (".60", ".600", ".3600", "")
+            row_name = ".".join(tags[len(prefix) : -len(suffix)])
+            table[row_name] = [counters[key + suffix] for suffix in TIME_SUFFIXES]
+
+    return table
+
+
 class StatsCmd(Subcmd):
     NAME = "stats"
     HELP = "Prints statistics information for eden"

eden/cli/test/stats_test.py

@@ -9,7 +9,7 @@ import unittest
 from io import StringIO
 
 from .. import stats_print
-from ..stats import DiagInfoCounters, get_hg_importer_counters, get_store_latency
+from ..stats import DiagInfoCounters, get_counter_table, get_store_latency
 
 
 class StatsTest(unittest.TestCase):
@@ -112,21 +112,6 @@ key                1              2              3              4
 
 
 class HgImporterStatsTest(unittest.TestCase):
-    def test_call_counts_are_zero_if_no_data_was_logged(self) -> None:
-        counters: DiagInfoCounters = {}
-        table = get_hg_importer_counters(counters)
-        metrics = [
-            "cat_file",
-            "fetch_tree",
-            "manifest",
-            "manifest_node_for_commit",
-            "prefetch_files",
-        ]
-        for metric in metrics:
-            self.assertEqual(
-                table.get(metric), [0, 0, 0, 0], f"Metric {metric} should be zero"
-            )
-
     def test_cat_file_call_counts_are_extracted_from_counters(self) -> None:
         counters: DiagInfoCounters = {
             "hg_importer.cat_file.count": 10,
@@ -134,7 +119,7 @@ class HgImporterStatsTest(unittest.TestCase):
             "hg_importer.cat_file.count.60": 1,
             "hg_importer.cat_file.count.600": 7,
         }
-        table = get_hg_importer_counters(counters)
+        table = get_counter_table(counters, ["hg_importer"], ["count"])
         self.assertEqual(table.get("cat_file"), [1, 7, 9, 10])
 
     def test_table_includes_unknown_counters(self) -> None:
@@ -144,7 +129,7 @@ class HgImporterStatsTest(unittest.TestCase):
             "hg_importer.dog_file.count.60": 10,
             "hg_importer.dog_file.count.600": 70,
         }
-        table = get_hg_importer_counters(counters)
+        table = get_counter_table(counters, ["hg_importer"], ["count"])
         self.assertEqual(table.get("dog_file"), [10, 70, 90, 100])
 
 

eden/fs/service/EdenServer.cpp

@@ -825,7 +825,8 @@ folly::Future<std::shared_ptr<EdenMount>> EdenServer::mount(
     optional<TakeoverData::MountInfo>&& optionalTakeover) {
   auto backingStore = getBackingStore(
       initialConfig->getRepoType(), initialConfig->getRepoSource());
-  auto objectStore = ObjectStore::create(getLocalStore(), backingStore);
+  auto objectStore =
+      ObjectStore::create(getLocalStore(), backingStore, getSharedStats());
 
 #if _WIN32
   // Create the EdenMount object and insert the mount into the mountPoints_ map.

eden/fs/store/ObjectStore.cpp

@@ -7,6 +7,7 @@
 #include "ObjectStore.h"
 
 #include <folly/Conv.h>
+#include <folly/Format.h>
 #include <folly/futures/Future.h>
 #include <folly/io/IOBuf.h>
 #include <folly/logging/xlog.h>
@@ -16,6 +17,7 @@
 #include "eden/fs/model/Tree.h"
 #include "eden/fs/store/BackingStore.h"
 #include "eden/fs/store/LocalStore.h"
+#include "eden/fs/tracing/EdenStats.h"
 
 using folly::Future;
 using folly::IOBuf;
@@ -29,17 +31,20 @@ namespace eden {
 
 std::shared_ptr<ObjectStore> ObjectStore::create(
     shared_ptr<LocalStore> localStore,
-    shared_ptr<BackingStore> backingStore) {
-  return std::shared_ptr<ObjectStore>{
-      new ObjectStore{std::move(localStore), std::move(backingStore)}};
+    shared_ptr<BackingStore> backingStore,
+    shared_ptr<EdenStats> stats) {
+  return std::shared_ptr<ObjectStore>{new ObjectStore{
+      std::move(localStore), std::move(backingStore), std::move(stats)}};
 }
 
 ObjectStore::ObjectStore(
     shared_ptr<LocalStore> localStore,
-    shared_ptr<BackingStore> backingStore)
+    shared_ptr<BackingStore> backingStore,
+    shared_ptr<EdenStats> stats)
     : metadataCache_{folly::in_place, kCacheSize},
       localStore_{std::move(localStore)},
-      backingStore_{std::move(backingStore)} {}
+      backingStore_{std::move(backingStore)},
+      stats_{std::move(stats)} {}
 
 ObjectStore::~ObjectStore() {}
 
@@ -83,34 +88,21 @@ Future<shared_ptr<const Tree>> ObjectStore::getTree(const Hash& id) const {
       });
 }
 
-Future<shared_ptr<const Blob>> ObjectStore::getBlob(const Hash& id) const {
-  return localStore_->getBlob(id).thenValue(
-      [id, self = shared_from_this()](shared_ptr<const Blob> blob) {
-        if (blob) {
-          // Not computing the BlobMetadata here because if the blob was found
-          // in the local store, the LocalStore probably also has the metadata
-          // already, and the caller may not even need the SHA-1 here. (If the
-          // caller needed the SHA-1, they would have called getBlobMetadata
-          // instead.)
-          XLOG(DBG4) << "blob " << id << "  found in local store";
-          return makeFuture(shared_ptr<const Blob>(std::move(blob)));
+Future<shared_ptr<const Tree>> ObjectStore::getTreeForCommit(
+    const Hash& commitID) const {
+  XLOG(DBG3) << "getTreeForCommit(" << commitID << ")";
+
+  return backingStore_->getTreeForCommit(commitID).thenValue(
+      [commitID](std::shared_ptr<const Tree> tree) {
+        if (!tree) {
+          throw std::domain_error(folly::to<string>(
+              "unable to import commit ", commitID.toString()));
         }
 
-        // Look in the BackingStore
-        return self->backingStore_->getBlob(id).thenValue(
-            [self, id](unique_ptr<const Blob> loadedBlob) {
-              if (!loadedBlob) {
-                XLOG(DBG2) << "unable to find blob " << id;
-                // TODO: Perhaps we should do some short-term negative caching?
-                throw std::domain_error(
-                    folly::to<string>("blob ", id.toString(), " not found"));
-              }
-
-              XLOG(DBG3) << "blob " << id << "  retrieved from backing store";
-              auto metadata = self->localStore_->putBlob(id, loadedBlob.get());
-              self->metadataCache_.wlock()->set(id, metadata);
-              return shared_ptr<const Blob>(std::move(loadedBlob));
-            });
+        // For now we assume that the BackingStore will insert the Tree into the
+        // LocalStore on its own, so we don't have to update the LocalStore
+        // ourselves here.
+        return tree;
       });
 }
 
@@ -130,42 +122,73 @@ folly::Future<folly::Unit> ObjectStore::prefetchBlobs(
   return backingStore_->prefetchBlobs(ids);
 }
 
-Future<shared_ptr<const Tree>> ObjectStore::getTreeForCommit(
-    const Hash& commitID) const {
-  XLOG(DBG3) << "getTreeForCommit(" << commitID << ")";
+Future<shared_ptr<const Blob>> ObjectStore::getBlob(const Hash& id) const {
+  auto self = shared_from_this();
 
-  return backingStore_->getTreeForCommit(commitID).thenValue(
-      [commitID](std::shared_ptr<const Tree> tree) {
-        if (!tree) {
-          throw std::domain_error(folly::to<string>(
-              "unable to import commit ", commitID.toString()));
-        }
+  return localStore_->getBlob(id).thenValue([id, self](
+                                                shared_ptr<const Blob> blob) {
+    if (blob) {
+      // Not computing the BlobMetadata here because if the blob was found
+      // in the local store, the LocalStore probably also has the metadata
+      // already, and the caller may not even need the SHA-1 here. (If the
+      // caller needed the SHA-1, they would have called getBlobMetadata
+      // instead.)
+      XLOG(DBG4) << "blob " << id << " found in local store";
+      self->updateBlobStats(true, false);
+      return makeFuture(shared_ptr<const Blob>(std::move(blob)));
+    }
 
-        // For now we assume that the BackingStore will insert the Tree into the
-        // LocalStore on its own, so we don't have to update the LocalStore
-        // ourselves here.
-        return tree;
-      });
+    // Look in the BackingStore
+    return self->backingStore_->getBlob(id).thenValue(
+        [self, id](unique_ptr<const Blob> loadedBlob) {
+          if (loadedBlob) {
+            XLOG(DBG3) << "blob " << id << "  retrieved from backing store";
+            self->updateBlobStats(false, true);
+            auto metadata = self->localStore_->putBlob(id, loadedBlob.get());
+            self->metadataCache_.wlock()->set(id, metadata);
+            return shared_ptr<const Blob>(std::move(loadedBlob));
+          }
+
+          XLOG(DBG2) << "unable to find blob " << id;
+          self->updateBlobStats(false, false);
+          // TODO: Perhaps we should do some short-term negative caching?
+          throw std::domain_error(
+              folly::to<string>("blob ", id.toString(), " not found"));
+        });
+  });
+}
+
+void ObjectStore::updateBlobStats(bool local, bool backing) const {
+#if defined(EDEN_HAVE_STATS)
+  ObjectStoreThreadStats& stats = stats_->getObjectStoreStatsForCurrentThread();
+  stats.getBlobFromLocalStore.addValue(local);
+  stats.getBlobFromBackingStore.addValue(backing);
+#endif
 }
 
 Future<BlobMetadata> ObjectStore::getBlobMetadata(const Hash& id) const {
-  // First, check the in-memory cache.
+  // Check in-memory cache
   {
     auto metadataCache = metadataCache_.wlock();
     auto cacheIter = metadataCache->find(id);
     if (cacheIter != metadataCache->end()) {
+      updateBlobMetadataStats(true, false, false);
       return cacheIter->second;
     }
   }
 
+  auto self = shared_from_this();
+
+  // Check local store
   return localStore_->getBlobMetadata(id).thenValue(
-      [id, self = shared_from_this()](std::optional<BlobMetadata>&& localData) {
-        if (localData.has_value()) {
-          self->metadataCache_.wlock()->set(id, localData.value());
-          return makeFuture(localData.value());
+      [self, id](std::optional<BlobMetadata>&& metadata) {
+        if (metadata) {
+          self->updateBlobMetadataStats(false, true, false);
+          self->metadataCache_.wlock()->set(id, *metadata);
+          return makeFuture(*metadata);
         }
 
-        // Load the blob from the BackingStore.
+        // Check backing store
         //
         // TODO: It would be nice to add a smarter API to the BackingStore so
         // that we can query it just for the blob metadata if it supports
@@ -175,41 +198,28 @@ Future<BlobMetadata> ObjectStore::getBlobMetadata(const Hash& id) const {
         // especially when we begin to expire entries in RocksDB.
         return self->backingStore_->getBlob(id).thenValue(
             [self, id](std::unique_ptr<Blob> blob) {
-              if (!blob) {
-                throw std::domain_error(
-                    folly::to<string>("blob ", id.toString(), " not found"));
+              if (blob) {
+                self->updateBlobMetadataStats(false, false, true);
+                auto metadata = self->localStore_->putBlob(id, blob.get());
+                self->metadataCache_.wlock()->set(id, metadata);
+                return makeFuture(metadata);
               }
 
-              auto metadata = self->localStore_->putBlob(id, blob.get());
-              self->metadataCache_.wlock()->set(id, metadata);
-              return makeFuture(metadata);
+              self->updateBlobMetadataStats(false, false, false);
+              throw std::domain_error(
+                  folly::to<string>("blob ", id.toString(), " not found"));
             });
       });
 }
 
-Future<uint64_t> ObjectStore::getBlobSize(const Hash& id) const {
-  // Check local store
-  auto self = shared_from_this();
-  return self->localStore_->getBlobSize(id).thenValue(
-      [self, id](std::optional<uint64_t> size) {
-        if (size) {
-          return makeFuture(*size);
-        }
-
-        // Check backing store for blob
-        return self->backingStore_->getBlob(id).thenValue(
-            [self, id](std::unique_ptr<Blob> blob) {
-              if (blob) {
-                uint64_t size = blob.get()->getSize();
-                self->localStore_->putBlobWithoutMetadata(id, blob.get());
-                self->localStore_->putBlobSize(id, size);
-                return makeFuture(size);
-              } else {
-                throw std::domain_error(
-                    folly::to<string>("blob ", id.toString(), " not found"));
-              }
-            });
-      });
+void ObjectStore::updateBlobMetadataStats(bool memory, bool local, bool backing)
+    const {
+#if defined(EDEN_HAVE_STATS)
+  ObjectStoreThreadStats& stats = stats_->getObjectStoreStatsForCurrentThread();
+  stats.getBlobMetadataFromMemory.addValue(memory);
+  stats.getBlobMetadataFromLocalStore.addValue(local);
+  stats.getBlobMetadataFromBackingStore.addValue(backing);
+#endif
 }
 
 Future<Hash> ObjectStore::getBlobSha1(const Hash& id) const {
@@ -217,5 +227,45 @@ Future<Hash> ObjectStore::getBlobSha1(const Hash& id) const {
       [](const BlobMetadata& metadata) { return metadata.sha1; });
 }
 
+Future<uint64_t> ObjectStore::getBlobSize(const Hash& id) const {
+  auto self = shared_from_this();
+
+  // Check local store for size
+  return self->localStore_->getBlobSize(id).thenValue(
+      [self, id](std::optional<uint64_t> size) {
+        if (size) {
+          self->updateBlobSizeStats(true, false);
+          self->localStore_->putBlobSize(id, *size);
+          return makeFuture(*size);
+        }
+
+        // Check backing store for blob
+        return self->backingStore_->getBlob(id).thenValue(
+            [self, id](std::unique_ptr<Blob> blob) {
+              if (blob) {
+                const uint64_t size = blob.get()->getSize();
+                self->updateBlobSizeStats(false, true);
+                self->localStore_->putBlobWithoutMetadata(id, blob.get());
+                self->localStore_->putBlobSize(id, size);
+
+                return makeFuture(size);
+              }
+
+              // Not found
+              self->updateBlobSizeStats(false, false);
+              throw std::domain_error(
+                  folly::to<string>("blob ", id.toString(), " not found"));
+            });
+      });
+}
+
+void ObjectStore::updateBlobSizeStats(bool local, bool backing) const {
+#if defined(EDEN_HAVE_STATS)
+  ObjectStoreThreadStats& stats = stats_->getObjectStoreStatsForCurrentThread();
+  stats.getBlobSizeFromLocalStore.addValue(local);
+  stats.getBlobSizeFromBackingStore.addValue(backing);
+#endif
+}
+
 } // namespace eden
 } // namespace facebook

eden/fs/store/ObjectStore.h

@@ -13,6 +13,7 @@
 #include "eden/fs/model/Hash.h"
 #include "eden/fs/store/BlobMetadata.h"
 #include "eden/fs/store/IObjectStore.h"
+#include "eden/fs/tracing/EdenStats.h"
 
 namespace facebook {
 namespace eden {
@@ -37,7 +38,8 @@ class ObjectStore : public IObjectStore,
  public:
   static std::shared_ptr<ObjectStore> create(
       std::shared_ptr<LocalStore> localStore,
-      std::shared_ptr<BackingStore> backingStore);
+      std::shared_ptr<BackingStore> backingStore,
+      std::shared_ptr<EdenStats> stats);
   ~ObjectStore() override;
 
   /**
@@ -50,18 +52,6 @@ class ObjectStore : public IObjectStore,
   folly::Future<std::shared_ptr<const Tree>> getTree(
       const Hash& id) const override;
 
-  /**
-   * Get a Blob by ID.
-   *
-   * This returns a Future object that will produce the Blob when it is ready.
-   * It may result in a std::domain_error if the specified blob ID does not
-   * exist, or possibly other exceptions on error.
-   */
-  folly::Future<std::shared_ptr<const Blob>> getBlob(
-      const Hash& id) const override;
-  folly::Future<folly::Unit> prefetchBlobs(
-      const std::vector<Hash>& ids) const override;
-
   /**
    * Get a commit's root Tree.
    *
@@ -72,6 +62,19 @@ class ObjectStore : public IObjectStore,
   folly::Future<std::shared_ptr<const Tree>> getTreeForCommit(
       const Hash& commitID) const override;
 
+  folly::Future<folly::Unit> prefetchBlobs(
+      const std::vector<Hash>& ids) const override;
+
+  /**
+   * Get a Blob by ID.
+   *
+   * This returns a Future object that will produce the Blob when it is ready.
+   * It may result in a std::domain_error if the specified blob ID does not
+   * exist, or possibly other exceptions on error.
+   */
+  folly::Future<std::shared_ptr<const Blob>> getBlob(
+      const Hash& id) const override;
+
   /**
    * Get metadata about a Blob.
    *
@@ -109,7 +112,8 @@ class ObjectStore : public IObjectStore,
   // Forbidden constructor. Use create().
   ObjectStore(
       std::shared_ptr<LocalStore> localStore,
-      std::shared_ptr<BackingStore> backingStore);
+      std::shared_ptr<BackingStore> backingStore,
+      std::shared_ptr<EdenStats> stats);
   // Forbidden copy constructor and assignment operator
   ObjectStore(ObjectStore const&) = delete;
   ObjectStore& operator=(ObjectStore const&) = delete;
@@ -146,6 +150,12 @@ class ObjectStore : public IObjectStore,
    * Multiple ObjectStores may share the same BackingStore.
    */
   std::shared_ptr<BackingStore> backingStore_;
+
+  std::shared_ptr<EdenStats> const stats_;
+
+  void updateBlobStats(bool local, bool backing) const;
+  void updateBlobSizeStats(bool local, bool backing) const;
+  void updateBlobMetadataStats(bool memory, bool local, bool backing) const;
 };
 
 } // namespace eden

eden/fs/store/hg/test/HgBackingStoreTest.cpp

@@ -51,7 +51,7 @@ struct HgBackingStoreTest : TestRepo, ::testing::Test {
   std::shared_ptr<HgBackingStore> backingStore{
       std::make_shared<HgBackingStore>(&importer, localStore.get(), stats)};
   std::shared_ptr<ObjectStore> objectStore{
-      ObjectStore::create(localStore, backingStore)};
+      ObjectStore::create(localStore, backingStore, stats)};
 };
 
 TEST_F(

eden/fs/store/hg/test/HgImportErrorsTest.cpp

@@ -159,7 +159,7 @@ class HgImportErrorTest : public ::testing::Test {
         fakeImportHelper);
     backingStore_ =
         make_shared<HgBackingStore>(importer_.get(), localStore_.get(), stats_);
-    objectStore_ = ObjectStore::create(localStore_, backingStore_);
+    objectStore_ = ObjectStore::create(localStore_, backingStore_, stats_);
   }
 
   template <typename ImporterType>

eden/fs/store/test/BlobAccessTest.cpp

@@ -65,7 +65,10 @@ struct BlobAccessTest : ::testing::Test {
   BlobAccessTest()
       : localStore{std::make_shared<NullLocalStore>()},
         backingStore{std::make_shared<FakeBackingStore>(localStore)},
-        objectStore{ObjectStore::create(localStore, backingStore)},
+        objectStore{ObjectStore::create(
+            localStore,
+            backingStore,
+            std::make_shared<EdenStats>())},
         blobCache{BlobCache::create(10, 0)},
         blobAccess{objectStore, blobCache} {
     backingStore->putBlob(hash3, "333"_sp)->setReady();

eden/fs/store/test/DiffTest.cpp

@@ -52,7 +52,8 @@ class DiffTest : public ::testing::Test {
   void SetUp() override {
     localStore_ = make_shared<MemoryLocalStore>();
     backingStore_ = make_shared<FakeBackingStore>(localStore_);
-    store_ = ObjectStore::create(localStore_, backingStore_);
+    store_ = ObjectStore::create(
+        localStore_, backingStore_, std::make_shared<EdenStats>());
   }
 
   Future<ScmStatus> diffCommits(StringPiece commit1, StringPiece commit2) {

eden/fs/store/test/ObjectStoreTest.cpp

@@ -20,7 +20,8 @@ class ObjectStoreTest : public ::testing::Test {
   void SetUp() override {
     localStore_ = std::make_shared<MemoryLocalStore>();
     backingStore_ = std::make_shared<FakeBackingStore>(localStore_);
-    objectStore_ = ObjectStore::create(localStore_, backingStore_);
+    stats_ = std::make_shared<EdenStats>();
+    objectStore_ = ObjectStore::create(localStore_, backingStore_, stats_);
   }
 
   Hash putReadyBlob(folly::StringPiece data) {
@@ -33,6 +34,7 @@ class ObjectStoreTest : public ::testing::Test {
 
   std::shared_ptr<LocalStore> localStore_;
   std::shared_ptr<FakeBackingStore> backingStore_;
+  std::shared_ptr<EdenStats> stats_;
   std::shared_ptr<ObjectStore> objectStore_;
 };
 
@@ -43,7 +45,7 @@ TEST_F(ObjectStoreTest, getBlobSizeFromLocalStore) {
   // Get blob size from backing store, caches in local store
   objectStore_->getBlobSize(id);
   // Clear backing store
-  objectStore_ = ObjectStore::create(localStore_, nullptr);
+  objectStore_ = ObjectStore::create(localStore_, nullptr, stats_);
 
   size_t expectedSize = data.size();
   size_t size = objectStore_->getBlobSize(id).get();

eden/fs/testharness/TestMount.cpp

@@ -179,7 +179,7 @@ void TestMount::createMountWithoutInitializing(
 
 void TestMount::createMount() {
   shared_ptr<ObjectStore> objectStore =
-      ObjectStore::create(localStore_, backingStore_);
+      ObjectStore::create(localStore_, backingStore_, stats_);
   edenMount_ = EdenMount::create(
       std::move(config_), std::move(objectStore), blobCache_, serverState_);
 }
@@ -225,6 +225,8 @@ void TestMount::initTestDirectory() {
   // Create localStore_ and backingStore_
   localStore_ = make_shared<MemoryLocalStore>();
   backingStore_ = make_shared<FakeBackingStore>(localStore_);
+
+  stats_ = make_shared<EdenStats>();
 }
 
 Dispatcher* TestMount::getDispatcher() const {
@@ -246,7 +248,7 @@ void TestMount::remount() {
   // Create a new copy of the CheckoutConfig
   auto config = make_unique<CheckoutConfig>(*edenMount_->getConfig());
   // Create a new ObjectStore pointing to our local store and backing store
-  auto objectStore = ObjectStore::create(localStore_, backingStore_);
+  auto objectStore = ObjectStore::create(localStore_, backingStore_, stats_);
 
   // Reset the edenMount_ pointer.  This will destroy the old EdenMount
   // assuming that no-one else still has any references to it.
@@ -269,7 +271,7 @@ void TestMount::remountGracefully() {
   // Create a new copy of the CheckoutConfig
   auto config = make_unique<CheckoutConfig>(*edenMount_->getConfig());
   // Create a new ObjectStore pointing to our local store and backing store
-  auto objectStore = ObjectStore::create(localStore_, backingStore_);
+  auto objectStore = ObjectStore::create(localStore_, backingStore_, stats_);
 
   auto takeoverData =
       edenMount_->shutdown(/*doTakeover=*/true, /*allowFuseNotStarted=*/true)

eden/fs/testharness/TestMount.h

@@ -341,6 +341,7 @@ class TestMount {
   std::shared_ptr<EdenMount> edenMount_;
   std::shared_ptr<LocalStore> localStore_;
   std::shared_ptr<FakeBackingStore> backingStore_;
+  std::shared_ptr<EdenStats> stats_;
   std::shared_ptr<BlobCache> blobCache_;
 
   /*

eden/fs/tracing/EdenStats.cpp

@@ -28,6 +28,10 @@ FuseThreadStats& EdenStats::getFuseStatsForCurrentThread() {
   return *threadLocalFuseStats_.get();
 }
 
+ObjectStoreThreadStats& EdenStats::getObjectStoreStatsForCurrentThread() {
+  return *threadLocalObjectStoreStats_.get();
+}
+
 HgBackingStoreThreadStats& EdenStats::getHgBackingStoreStatsForCurrentThread() {
   return *threadLocalHgBackingStoreStats_.get();
 }
@@ -40,6 +44,9 @@ void EdenStats::aggregate() {
   for (auto& stats : threadLocalFuseStats_.accessAllThreads()) {
     stats.aggregate();
   }
+  for (auto& stats : threadLocalObjectStoreStats_.accessAllThreads()) {
+    stats.aggregate();
+  }
   for (auto& stats : threadLocalHgBackingStoreStats_.accessAllThreads()) {
     stats.aggregate();
   }
@@ -74,6 +81,7 @@ EdenThreadStatsBase::Timeseries EdenThreadStatsBase::createTimeseries(
     const std::string& name) {
   auto timeseries = Timeseries{this, name};
   timeseries.exportStat(facebook::stats::COUNT);
+  timeseries.exportStat(facebook::stats::PERCENT);
   return timeseries;
 }
 #endif

eden/fs/tracing/EdenStats.h

@@ -8,6 +8,7 @@
 
 #include <folly/ThreadLocal.h>
 #include <memory>
+
 #include "common/stats/ThreadLocalStats.h"
 #include "eden/fs/eden-config.h"
 
@@ -15,6 +16,7 @@ namespace facebook {
 namespace eden {
 
 class FuseThreadStats;
+class ObjectStoreThreadStats;
 class HgBackingStoreThreadStats;
 class HgImporterThreadStats;
 
@@ -27,6 +29,13 @@ class EdenStats {
    */
   FuseThreadStats& getFuseStatsForCurrentThread();
 
+  /**
+   * This function can be called on any thread.
+   *
+   * The returned object can be used only on the current thread.
+   */
+  ObjectStoreThreadStats& getObjectStoreStatsForCurrentThread();
+
   /**
    * This function can be called on any thread.
    *
@@ -51,6 +60,8 @@ class EdenStats {
 
   folly::ThreadLocal<FuseThreadStats, ThreadLocalTag, void>
       threadLocalFuseStats_;
+  folly::ThreadLocal<ObjectStoreThreadStats, ThreadLocalTag, void>
+      threadLocalObjectStoreStats_;
   folly::ThreadLocal<HgBackingStoreThreadStats, ThreadLocalTag, void>
       threadLocalHgBackingStoreStats_;
   folly::ThreadLocal<HgImporterThreadStats, ThreadLocalTag, void>
@@ -143,6 +154,31 @@ class FuseThreadStats : public EdenThreadStatsBase {
       std::chrono::seconds now);
 };
 
+/**
+ * @see ObjectStore
+ */
+class ObjectStoreThreadStats : public EdenThreadStatsBase {
+ public:
+#if defined(EDEN_HAVE_STATS)
+  Timeseries getBlobFromLocalStore{
+      createTimeseries("object_store.get_blob.local_store")};
+  Timeseries getBlobFromBackingStore{
+      createTimeseries("object_store.get_blob.backing_store")};
+
+  Timeseries getBlobMetadataFromMemory{
+      createTimeseries("object_store.get_blob_metadata.memory")};
+  Timeseries getBlobMetadataFromLocalStore{
+      createTimeseries("object_store.get_blob_metadata.local_store")};
+  Timeseries getBlobMetadataFromBackingStore{
+      createTimeseries("object_store.get_blob_metadata.backing_store")};
+
+  Timeseries getBlobSizeFromLocalStore{
+      createTimeseries("object_store.get_blob_size.local_store")};
+  Timeseries getBlobSizeFromBackingStore{
+      createTimeseries("object_store.get_blob_size.backing_store")};
+#endif
+};
+
 /**
  * @see HgBackingStore
  */

eden/integration/rocksdb_store_test.py

@@ -115,8 +115,8 @@ ephemeral-size-limit = "1"
                 counters.get("local_store.auto_gc.last_duration_ms"), 0
             )
 
-        # Run "eden stats local_store" and check the output
-        stats_output = self.eden.run_cmd("stats", "local_store")
+        # Run "eden stats local-store" and check the output
+        stats_output = self.eden.run_cmd("stats", "local-store")
         print(stats_output)
         m = re.search(r"Successful Auto-GC Runs:\s+(\d+)", stats_output)
         self.assertIsNotNone(m)

eden/integration/stats_test.py

@@ -77,6 +77,43 @@ class FUSEStatsTest(testcase.EdenRepoTest):
                 continue
 
 
+class ObjectStoreStatsTest(testcase.EdenRepoTest):
+    def create_repo(self, name: str) -> HgRepository:
+        return self.create_hg_repo(name)
+
+    def populate_repo(self) -> None:
+        self.repo.write_file("foo.txt", "foo\n")
+        self.repo.commit("Initial commit.")
+
+    def test_get_blob(self) -> None:
+        TEMPLATE = "object_store.get_blob.{}_store.pct"
+        LOCAL = TEMPLATE.format("local")
+        BACKING = TEMPLATE.format("backing")
+
+        counters = self.get_counters()
+        self.assertEqual(counters.get(LOCAL, 0) + counters.get(BACKING, 0), 0)
+
+        foo = Path(self.mount) / "foo.txt"
+        foo.read_bytes()
+
+        counters = self.get_counters()
+        self.assertEqual(counters.get(LOCAL, 0) + counters.get(BACKING, 0), 100)
+
+    def test_get_blob_size(self) -> None:
+        TEMPLATE = "object_store.get_blob_size.{}_store.pct"
+        LOCAL = TEMPLATE.format("local")
+        BACKING = TEMPLATE.format("backing")
+
+        counters = self.get_counters()
+        self.assertEqual(counters.get(LOCAL, 0) + counters.get(BACKING, 0), 0)
+
+        foo = Path(self.mount) / "foo.txt"
+        foo.stat()
+
+        counters = self.get_counters()
+        self.assertEqual(counters.get(LOCAL, 0) + counters.get(BACKING, 0), 100)
+
+
 class HgBackingStoreStatsTest(testcase.EdenRepoTest):
     def test_reading_file_gets_file_from_hg(self) -> None:
         counters_before = self.get_counters()