debugging: Implement "debugdetectissues" command for detecting signs of repository issues

Summary:
Implements a "debugdetectissues" command, which runs a series of checks on the repository meant to detect potential issues, logging the data to Scuba so that we can determine how common certain issues are, and under what conditions they appear.

Future extensions of this change may involve merging the functionality into hg doctor, and setting the command to run automatically in the background on some interval.

Reviewed By: DurhamG

Differential Revision: D21558170

fbshipit-source-id: a878ae1804d5f11c83a574e0dc3c802b564d2ced

eden/scm/edenscm/hgext/rage.py

@@ -29,6 +29,7 @@ from edenscm.mercurial import (
     cmdutil,
     color,
     commands,
+    detectissues,
     encoding,
     error,
     extensions,
@@ -331,6 +332,7 @@ def _makerage(ui, repo, **opts):
         ("hg config (local)", lambda: "\n".join(localconfig(ui))),
         ("hg sparse", lambda: hgcmd("sparse")),
         ("hg debuginstall", lambda: hgcmd("debuginstall")),
+        ("hg debugdetectissues", lambda: hgcmd("debugdetectissues")),
         ("usechg", usechginfo),
         (
             "uptime",

eden/scm/edenscm/mercurial/commands/debug.py

@@ -37,6 +37,7 @@ from .. import (
     context,
     dagparser,
     dagutil,
+    detectissues,
     drawdag,
     edenfs,
     encoding,
@@ -495,6 +496,22 @@ def debugcolor(ui, **opts):
         return _debugdisplaycolor(ui)
 
 
+@command("debugdetectissues", [], "")
+def debugdetectissues(ui, repo):
+    """various repository integrity and health checks. for automatic remediation, use doctor."""
+
+    findings = detectissues.detectissues(repo)
+
+    for detectorname, issues in findings.items():
+        ui.write(
+            _("ran issue detector '%s', found %s issues\n")
+            % (detectorname, len(issues))
+        )
+        for issue in issues:
+            ui.write(_("'%s': '%s'\n") % (issue.category, issue.message))
+            ui.log("repoissues", issue.message, category=issue.category, **issue.data)
+
+
 def _debugdisplaycolor(ui):
     ui = ui.copy()
     ui._styles.clear()

eden/scm/edenscm/mercurial/detectissues.py

@@ -0,0 +1,110 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This software may be used and distributed according to the terms of the
+# GNU General Public License version 2.
+
+# detectissues.py - detect various issues with the repository
+
+
+from __future__ import absolute_import
+
+import os
+
+from .i18n import _
+from .pycompat import ossep
+
+
+class issue(object):
+    def __init__(self, category, message, data):
+        self.category = category
+        self.message = message
+        self.data = data
+
+
+def computecachesize(repo):
+    """measure size of cache directory"""
+    from ..hgext.remotefilelog import shallowutil
+
+    cachepath = shallowutil.getcachepath(repo.ui)
+
+    skipped = 0
+
+    cachesize = 0
+    manifestsize = 0
+    for root, dirs, files in os.walk(cachepath):
+        dirsize = 0
+        for filename in files:
+            try:
+                stat = os.lstat(os.path.join(root, filename))
+                dirsize += stat.st_size
+            except Exception as e:
+                repo.ui.warn(
+                    _("error statting file '%s': %r. skipping file.\n") % (filename, e)
+                )
+                skipped += 1
+
+        relpath = os.path.relpath(root, cachepath)
+        segments = relpath.split(ossep)
+        if "manifests" in segments[1:]:
+            manifestsize += dirsize
+        else:
+            cachesize += dirsize
+
+    return (cachesize, manifestsize, skipped)
+
+
+def cachesizeexceedslimit(repo):
+    cachelimit = repo.ui.configbytes("remotefilelog", "cachelimit", "10GB")
+    manifestlimit = repo.ui.configbytes("remotefilelog", "manifestlimit", "2GB")
+    cachesize, manifestsize, skipped = computecachesize(repo)
+    issues = []
+    if cachesize > cachelimit:
+        issues.append(
+            issue(
+                "cache_size_exceeds_limit",
+                _("cache size of %s exceeds configured limit of %s. %s files skipped.")
+                % (cachesize, cachelimit, skipped),
+                {
+                    "cachesize": cachesize,
+                    "manifestsize": manifestsize,
+                    "cachelimit": cachelimit,
+                    "manifestlimit": manifestlimit,
+                    "skippedfiles": skipped,
+                },
+            )
+        )
+    if manifestsize > manifestlimit:
+        issues.append(
+            issue(
+                "manifest_size_exceeds_limit",
+                _(
+                    "manifest cache size of %s exceeds configured limit of %s. %s files skipped."
+                )
+                % (manifestsize, manifestlimit, skipped),
+                {
+                    "cachesize": cachesize,
+                    "manifestsize": manifestsize,
+                    "cachelimit": cachelimit,
+                    "manifestlimit": manifestlimit,
+                    "skippedfiles": skipped,
+                },
+            )
+        )
+    return issues
+
+
+def detectissues(repo):
+    issuedetectors = [cachesizeexceedslimit]
+
+    issues = {}
+    for func in issuedetectors:
+        name = func.__name__
+        try:
+            issues[name] = func(repo)
+        except Exception as e:
+            repo.ui.warn(
+                _("exception %r while running issue detector %s, skipping\n")
+                % (e, name)
+            )
+
+    return issues

eden/scm/tests/test-completion.t

@@ -104,6 +104,7 @@ Show debug commands if there are no other candidates
   debugdata
   debugdate
   debugdeltachain
+  debugdetectissues
   debugdifftree
   debugdirs
   debugdirstate
@@ -309,6 +310,7 @@ Show all commands + options
   debugdata: changelog, manifest, dir
   debugdate: extended, range
   debugdeltachain: changelog, manifest, dir, template
+  debugdetectissues: 
   debugdifftree: rev, include, exclude, style, template
   debugdirs: rev, print0
   debugdirstate: nodates, datesort, json

eden/scm/tests/test-fb-hgext-debugdetectissues.t

@@ -0,0 +1,26 @@
+  $ configure modern
+
+  $ newserver master
+  $ cat >> .hg/hgrc <<EOF
+  > [remotefilelog]
+  > cachelimit = 0B
+  > manifestlimit = 0B
+  > EOF
+  $ hg debugdetectissues
+  ran issue detector 'cachesizeexceedslimit', found 0 issues
+  $ echo "a" > a ; hg add a ; hg commit -qAm a
+  $ echo "b" > b ; hg add b ; hg commit -qAm b
+  $ hg debugdetectissues
+  ran issue detector 'cachesizeexceedslimit', found 0 issues
+  $ cd ..
+  $ clone master shallow
+  $ cd shallow
+  $ cat >> .hg/hgrc <<EOF
+  > [remotefilelog]
+  > cachelimit = 0B
+  > manifestlimit = 0B
+  > EOF
+  $ hg debugdetectissues
+  ran issue detector 'cachesizeexceedslimit', found 2 issues
+  'cache_size_exceeds_limit': 'cache size of 2610 exceeds configured limit of 0. 0 files skipped.'
+  'manifest_size_exceeds_limit': 'manifest cache size of 2426 exceeds configured limit of 0. 0 files skipped.'

eden/scm/tests/test-help.t

@@ -966,6 +966,9 @@ Test list of internal help commands
    debugdate     parse and display a date
    debugdeltachain
                  dump information about delta chains in a revlog
+   debugdetectissues
+                 various repository integrity and health checks. for automatic
+                 remediation, use doctor.
    debugdifftree
                  diff two trees
    debugdirs     list directories