From 341fbdc1aa8c66d9f6360566f5fbed46069aa7c1 Mon Sep 17 00:00:00 2001 From: Meyer Jacobs Date: Tue, 19 May 2020 18:09:41 -0700 Subject: [PATCH] debugging: Implement "debugdetectissues" command for detecting signs of repository issues Summary: Implements a "debugdetectissues" command, which runs a series of checks on the repository meant to detect potential issues, logging the data to Scuba so that we can determine how common certain issues are, and under what conditions they appear. Future extensions of this change may involve merging the functionality into hg doctor, and setting the command to run automatically in the background on some interval. Reviewed By: DurhamG Differential Revision: D21558170 fbshipit-source-id: a878ae1804d5f11c83a574e0dc3c802b564d2ced --- eden/scm/edenscm/hgext/rage.py | 2 + eden/scm/edenscm/mercurial/commands/debug.py | 17 +++ eden/scm/edenscm/mercurial/detectissues.py | 110 ++++++++++++++++++ eden/scm/tests/test-completion.t | 2 + .../tests/test-fb-hgext-debugdetectissues.t | 26 +++++ eden/scm/tests/test-help.t | 3 + 6 files changed, 160 insertions(+) create mode 100644 eden/scm/edenscm/mercurial/detectissues.py create mode 100644 eden/scm/tests/test-fb-hgext-debugdetectissues.t diff --git a/eden/scm/edenscm/hgext/rage.py b/eden/scm/edenscm/hgext/rage.py index 39e6eb00f3..0305ec4be5 100644 --- a/eden/scm/edenscm/hgext/rage.py +++ b/eden/scm/edenscm/hgext/rage.py @@ -29,6 +29,7 @@ from edenscm.mercurial import ( cmdutil, color, commands, + detectissues, encoding, error, extensions, @@ -331,6 +332,7 @@ def _makerage(ui, repo, **opts): ("hg config (local)", lambda: "\n".join(localconfig(ui))), ("hg sparse", lambda: hgcmd("sparse")), ("hg debuginstall", lambda: hgcmd("debuginstall")), + ("hg debugdetectissues", lambda: hgcmd("debugdetectissues")), ("usechg", usechginfo), ( "uptime", diff --git a/eden/scm/edenscm/mercurial/commands/debug.py b/eden/scm/edenscm/mercurial/commands/debug.py index 18a5398aa8..3c576ef3c9 100644 --- a/eden/scm/edenscm/mercurial/commands/debug.py +++ b/eden/scm/edenscm/mercurial/commands/debug.py @@ -37,6 +37,7 @@ from .. import ( context, dagparser, dagutil, + detectissues, drawdag, edenfs, encoding, @@ -495,6 +496,22 @@ def debugcolor(ui, **opts): return _debugdisplaycolor(ui) +@command("debugdetectissues", [], "") +def debugdetectissues(ui, repo): + """various repository integrity and health checks. for automatic remediation, use doctor.""" + + findings = detectissues.detectissues(repo) + + for detectorname, issues in findings.items(): + ui.write( + _("ran issue detector '%s', found %s issues\n") + % (detectorname, len(issues)) + ) + for issue in issues: + ui.write(_("'%s': '%s'\n") % (issue.category, issue.message)) + ui.log("repoissues", issue.message, category=issue.category, **issue.data) + + def _debugdisplaycolor(ui): ui = ui.copy() ui._styles.clear() diff --git a/eden/scm/edenscm/mercurial/detectissues.py b/eden/scm/edenscm/mercurial/detectissues.py new file mode 100644 index 0000000000..059550ca1d --- /dev/null +++ b/eden/scm/edenscm/mercurial/detectissues.py @@ -0,0 +1,110 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2. + +# detectissues.py - detect various issues with the repository + + +from __future__ import absolute_import + +import os + +from .i18n import _ +from .pycompat import ossep + + +class issue(object): + def __init__(self, category, message, data): + self.category = category + self.message = message + self.data = data + + +def computecachesize(repo): + """measure size of cache directory""" + from ..hgext.remotefilelog import shallowutil + + cachepath = shallowutil.getcachepath(repo.ui) + + skipped = 0 + + cachesize = 0 + manifestsize = 0 + for root, dirs, files in os.walk(cachepath): + dirsize = 0 + for filename in files: + try: + stat = os.lstat(os.path.join(root, filename)) + dirsize += stat.st_size + except Exception as e: + repo.ui.warn( + _("error statting file '%s': %r. skipping file.\n") % (filename, e) + ) + skipped += 1 + + relpath = os.path.relpath(root, cachepath) + segments = relpath.split(ossep) + if "manifests" in segments[1:]: + manifestsize += dirsize + else: + cachesize += dirsize + + return (cachesize, manifestsize, skipped) + + +def cachesizeexceedslimit(repo): + cachelimit = repo.ui.configbytes("remotefilelog", "cachelimit", "10GB") + manifestlimit = repo.ui.configbytes("remotefilelog", "manifestlimit", "2GB") + cachesize, manifestsize, skipped = computecachesize(repo) + issues = [] + if cachesize > cachelimit: + issues.append( + issue( + "cache_size_exceeds_limit", + _("cache size of %s exceeds configured limit of %s. %s files skipped.") + % (cachesize, cachelimit, skipped), + { + "cachesize": cachesize, + "manifestsize": manifestsize, + "cachelimit": cachelimit, + "manifestlimit": manifestlimit, + "skippedfiles": skipped, + }, + ) + ) + if manifestsize > manifestlimit: + issues.append( + issue( + "manifest_size_exceeds_limit", + _( + "manifest cache size of %s exceeds configured limit of %s. %s files skipped." + ) + % (manifestsize, manifestlimit, skipped), + { + "cachesize": cachesize, + "manifestsize": manifestsize, + "cachelimit": cachelimit, + "manifestlimit": manifestlimit, + "skippedfiles": skipped, + }, + ) + ) + return issues + + +def detectissues(repo): + issuedetectors = [cachesizeexceedslimit] + + issues = {} + for func in issuedetectors: + name = func.__name__ + try: + issues[name] = func(repo) + except Exception as e: + repo.ui.warn( + _("exception %r while running issue detector %s, skipping\n") + % (e, name) + ) + + return issues diff --git a/eden/scm/tests/test-completion.t b/eden/scm/tests/test-completion.t index e1cd4ba261..5ac57dffbe 100644 --- a/eden/scm/tests/test-completion.t +++ b/eden/scm/tests/test-completion.t @@ -104,6 +104,7 @@ Show debug commands if there are no other candidates debugdata debugdate debugdeltachain + debugdetectissues debugdifftree debugdirs debugdirstate @@ -309,6 +310,7 @@ Show all commands + options debugdata: changelog, manifest, dir debugdate: extended, range debugdeltachain: changelog, manifest, dir, template + debugdetectissues: debugdifftree: rev, include, exclude, style, template debugdirs: rev, print0 debugdirstate: nodates, datesort, json diff --git a/eden/scm/tests/test-fb-hgext-debugdetectissues.t b/eden/scm/tests/test-fb-hgext-debugdetectissues.t new file mode 100644 index 0000000000..1b4a18c397 --- /dev/null +++ b/eden/scm/tests/test-fb-hgext-debugdetectissues.t @@ -0,0 +1,26 @@ + $ configure modern + + $ newserver master + $ cat >> .hg/hgrc < [remotefilelog] + > cachelimit = 0B + > manifestlimit = 0B + > EOF + $ hg debugdetectissues + ran issue detector 'cachesizeexceedslimit', found 0 issues + $ echo "a" > a ; hg add a ; hg commit -qAm a + $ echo "b" > b ; hg add b ; hg commit -qAm b + $ hg debugdetectissues + ran issue detector 'cachesizeexceedslimit', found 0 issues + $ cd .. + $ clone master shallow + $ cd shallow + $ cat >> .hg/hgrc < [remotefilelog] + > cachelimit = 0B + > manifestlimit = 0B + > EOF + $ hg debugdetectissues + ran issue detector 'cachesizeexceedslimit', found 2 issues + 'cache_size_exceeds_limit': 'cache size of 2610 exceeds configured limit of 0. 0 files skipped.' + 'manifest_size_exceeds_limit': 'manifest cache size of 2426 exceeds configured limit of 0. 0 files skipped.' diff --git a/eden/scm/tests/test-help.t b/eden/scm/tests/test-help.t index a68e04bcfb..71d7666d78 100644 --- a/eden/scm/tests/test-help.t +++ b/eden/scm/tests/test-help.t @@ -966,6 +966,9 @@ Test list of internal help commands debugdate parse and display a date debugdeltachain dump information about delta chains in a revlog + debugdetectissues + various repository integrity and health checks. for automatic + remediation, use doctor. debugdifftree diff two trees debugdirs list directories