Add optional cache validation

Summary:
There are reports of the local cache becoming invalid when stored on disk. This
adds an option that will do some basic validation and remediation for those
entries, and log some data to disk.

This is optional, since it incurs some performance overhead. We just want to use
it long enough to track down the issue.

Test Plan: Added a test

Reviewers: sid0, pyd, ericsumner, rmcelroy, mitrandir

Reviewed By: mitrandir

Differential Revision: https://phabricator.fb.com/D1774724

Signature: t1:1774724:1420827432:06ace9d1dc078f469e0f61ebd7f604fc3b606f6d

remotefilelog/fileserverclient.py

@@ -306,6 +306,7 @@ class localcache(object):
         self.ui = repo.ui
         self.repo = repo
         self.cachepath = self.ui.config("remotefilelog", "cachepath")
+        self._validatecachelog = self.ui.config("remotefilelog", "validatecachelog")
         if self.cachepath:
             self.cachepath = util.expandpath(self.cachepath)
         self.uid = os.getuid()
@@ -326,7 +327,12 @@ class localcache(object):
 
     def __contains__(self, key):
         path = os.path.join(self.cachepath, key)
-        return os.path.exists(path)
+        exists = os.path.exists(path)
+        if exists and self._validatecachelog and not self._validatekey(path,
+            'contains'):
+            return False
+
+        return exists
 
     def write(self, key, data):
         path = os.path.join(self.cachepath, key)
@@ -342,6 +348,10 @@ class localcache(object):
             if f:
                 f.close()
 
+        if self._validatecachelog:
+            if not self._validatekey(path, 'write'):
+                raise util.Abort(_("local cache write was corrupted %s") % path)
+
         stat = os.stat(path)
         if stat.st_uid == self.uid:
             os.chmod(path, 0o0664)
@@ -355,12 +365,43 @@ class localcache(object):
             # we should never have empty files
             if not result:
                 os.remove(path)
-                raise KeyError("empty local cache file")
+                raise KeyError("empty local cache file %s" % path)
+
+            if self._validatecachelog and not self._validatedata(result):
+                with open(self._validatecachelog, 'a+') as f:
+                    f.write("corrupt %s during read\n" % path)
+                raise KeyError("corrupt local cache file %s" % path)
 
             return result
         except IOError:
             raise KeyError("key not in local cache")
 
+    def _validatekey(self, path, action):
+        with open(path, 'r') as f:
+            data = f.read()
+
+        if self._validatedata(data):
+            return True
+
+        with open(self._validatecachelog, 'a+') as f:
+            f.write("corrupt %s during %s\n" % (path, action))
+
+        os.rename(path, path + ".corrupt")
+        return False
+
+    def _validatedata(self, data):
+        try:
+            if len(data) > 0:
+                size = data.split('\0', 1)[0]
+                size = int(size)
+                if size < len(data):
+                    # The data looks to be well formed.
+                    return True
+        except ValueError:
+            pass
+
+        return False
+
     def markrepo(self):
         repospath = os.path.join(self.cachepath, "repos")
         with open(repospath, 'a') as reposfile:

tests/test-corrupt-cache.t

@@ -24,3 +24,15 @@ Verify corrupt cache error message
   $ hg up tip 2>&1 | grep "corrupt cache data for"
       raise Exception("corrupt cache data for '%s'" % (self.filename))
   Exception: corrupt cache data for 'x'
+
+Verify detection and remediation when remotefilelog.validatecachelog is set
+
+  $ cat >> .hg/hgrc <<EOF
+  > [remotefilelog]
+  > validatecachelog=$PWD/.hg/remotefilelog_cache.log
+  > EOF
+  $ hg up tip
+  3 files updated, 0 files merged, 0 files removed, 0 files unresolved
+  1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
+  $ cat .hg/remotefilelog_cache.log
+  corrupt $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/1406e74118627694268417491f018a4a883152f0 during contains