verify: add sqlverify command

Summary:
Adds an 'hg sqlverify' command that checks that the on disk revlogs match the
revlog data in mysql.

Test Plan: Added a test. Also ran it against internal repos to find issues.

Reviewers: #mercurial, quark

Reviewed By: quark

Subscribers: stash, quark

Differential Revision: https://phabricator.intern.facebook.com/D4847577

Tasks: 17067343

Signature: t1:4847577:1491520349:58d85ad98b073d03bc38e8efd1190b7e9a9ff821

hgsql.py

@@ -1690,3 +1690,85 @@ def _sqlreplay(repo, startrev, endrev):
             lock.release()
         if wlock:
             wlock.release()
+
+@command('^sqlverify', [
+    ('', 'earliest-rev', '', _('the earliest rev to process'), ''),
+    ], _('hg sqlverify'))
+def sqlverify(ui, repo, *args, **opts):
+    """verifies the current revlog indexes match the data in mysql
+
+    Runs in reverse order, so it verifies the latest commits first.
+    """
+    maxrev = len(repo.changelog) - 1
+    minrev = int(opts.get('earliest_rev') or '0')
+
+    def _helper():
+        stepsize = 1000
+        firstrev = max(minrev, maxrev - stepsize)
+        lastrev = maxrev
+
+        ui.progress('verifying', 0, total=maxrev - minrev)
+        while True:
+            _sqlverify(repo, firstrev, lastrev)
+            ui.progress('verifying', maxrev - firstrev, total=maxrev - minrev)
+            if firstrev == minrev:
+                break
+            lastrev = firstrev - 1
+            firstrev = max(minrev, firstrev - stepsize)
+
+        ui.progress('verifying', None)
+        ui.status("Verification passed\n")
+
+    executewithsql(repo, _helper, False)
+
+def _sqlverify(repo, minrev, maxrev):
+    queue = Queue.Queue()
+    abort = threading.Event()
+    t = threading.Thread(target=repo.fetchthread,
+        args=(queue, abort, minrev, maxrev))
+    t.setDaemon(True)
+
+    try:
+        t.start()
+
+        while True:
+            revdata = queue.get()
+            if not revdata:
+                return
+
+            # The background thread had an exception, rethrow from the
+            # foreground thread.
+            if isinstance(revdata, Exception):
+                raise revdata
+
+            # Validate revdata
+            path = revdata[0]
+            linkrev = revdata[3]
+            packedentry = revdata[4]
+
+            sqlentry = struct.unpack(revlog.indexformatng, packedentry)
+
+            rl = None
+            if path == '00changelog.i':
+                rl = repo.changelog
+            elif path == '00manifest.i':
+                rl = repo.manifestlog._revlog
+            else:
+                rl = revlog.revlog(repo.svfs, path)
+
+            node = sqlentry[7]
+            rev = rl.rev(node)
+            if rev == 0:
+                # The first entry has special whole-revlog flags in place of the
+                # offset in entry[0] that are not returned from revlog.index,
+                # so strip that data.
+                type = revlog.gettype(sqlentry[0])
+                sqlentry = (revlog.offset_type(0, type),) + sqlentry[1:]
+
+            revlogentry = rl.index[rev]
+            if revlogentry != sqlentry:
+                raise CorruptionException(("'%s:%s' with linkrev %s, disk does "
+                                           "not match mysql") %
+                                          (path, hex(node), str(linkrev)))
+    finally:
+        abort.set()

tests/test-sqlverify.t

@@ -0,0 +1,34 @@
+  $ . "$TESTDIR/library.sh"
+
+Populate the db with an initial commit
+
+  $ initserver master masterrepo
+  $ cd master
+  $ echo a > a
+  $ hg commit -Aqm 'add a'
+  $ echo b > b
+  $ hg commit -Aqm 'add b'
+  $ hg up -q 0
+  $ echo c > c
+  $ hg commit -Aqm 'add c'
+
+Run with a correct revlog
+
+  $ hg sqlverify
+  Verification passed
+
+Run with incorrect local revlogs
+
+  $ hg strip -r 1 --config hgsql.bypass=True
+  saved backup bundle to $TESTTMP/master/.hg/strip-backup/7c3bad9141dc-81844e36-backup.hg (glob)
+  $ hg unbundle --config hgsql.bypass=True $TESTTMP/master/.hg/strip-backup/7c3bad9141dc-81844e36-backup.hg
+  adding changesets
+  adding manifests
+  adding file changes
+  added 1 changesets with 1 changes to 1 files (+1 heads)
+  (run 'hg heads' to see heads, 'hg merge' to merge)
+
+  $ hg log -r tip --forcesync -T '{desc}\n'
+  add b
+  $ hg sqlverify 2>&1 | grep Corruption
+  hgext_hgsql.CorruptionException: '*' with linkrev *, disk does not match mysql (glob)