remotefilelog: enable background prefetch for pull

Summary:
Previously prefetch on pull ran in foreground. Now it can run in background
if specified. Optionally background prefetch can be followed by background
repack.

Test Plan: Added a test case

Reviewers: simonfar, durham

Reviewed By: simonfar

Subscribers: medson, mjpieters, #mercurial

Differential Revision: https://phabricator.intern.facebook.com/D5406626

Tasks: 19727343

Signature: t1:5406626:1500059334:207b4100cca536cbe33f6c6dfad596d03a6fa14f

remotefilelog/__init__.py

@@ -11,6 +11,7 @@ only download them ondemand as needed.
 Configs:
 
     ``packs.maxchainlen`` specifies the maximum delta chain length in pack files
+    ``remotefilelog.backgroundprefetch`` runs prefetch in background when True
 """
 
 from . import fileserverclient, remotefilelog, remotefilectx, shallowstore
@@ -744,14 +745,24 @@ def pull(orig, ui, repo, *pats, **opts):
     if shallowrepo.requirement in repo.requirements:
         # prefetch if it's configured
         prefetchrevset = ui.config('remotefilelog', 'pullprefetch', None)
+        bgrepack = repo.ui.configbool('remotefilelog',
+                                      'backgroundrepack', False)
+        bgprefetch = repo.ui.configbool('remotefilelog',
+                                        'backgroundprefetch', False)
+
         if prefetchrevset:
             ui.status(_("prefetching file contents\n"))
             revs = repo.revs(prefetchrevset)
             base = repo['.'].rev()
-            repo.prefetch(revs, base=base)
-
-        if repo.ui.configbool('remotefilelog', 'backgroundrepack'):
+            if bgprefetch:
+                repo.backgroundprefetch(prefetchrevset, repack=bgrepack)
+            else:
+                repo.prefetch(revs, base=base)
+                if bgrepack:
+                    repackmod.backgroundrepack(repo, incremental=True)
+        elif bgrepack:
             repackmod.backgroundrepack(repo, incremental=True)
+
     return result
 
 def exchangepull(orig, repo, remote, *args, **kwargs):
@@ -829,8 +840,14 @@ def debughistorypack(ui, path, **opts):
 def debugwaitonrepack(ui, repo, **opts):
     return debugcommands.debugwaitonrepack(repo)
 
+@command('debugwaitonprefetch', [
+    ], _('hg debugwaitonprefetch'))
+def debugwaitonprefetch(ui, repo, **opts):
+    return debugcommands.debugwaitonprefetch(repo)
+
 @command('prefetch', [
     ('r', 'rev', [], _('prefetch the specified revisions'), _('REV')),
+    ('', 'repack', False, _('run repack after prefetch')),
     ] + commands.walkopts, _('hg prefetch [OPTIONS] [FILE...]'))
 def prefetch(ui, repo, *pats, **opts):
     """prefetch file revisions from the server
@@ -850,7 +867,7 @@ def prefetch(ui, repo, *pats, **opts):
 
     revs = scmutil.revrange(repo, opts.get('rev'))
 
-    repo.prefetch(revs, pats=pats, opts=opts)
+    repo.prefetch(revs, repack=opts.get('repack'), pats=pats, opts=opts)
 
 @command('repack', [
      ('', 'background', None, _('run in a background process'), None),

remotefilelog/debugcommands.py

@@ -272,3 +272,8 @@ def debugwaitonrepack(repo):
     with repo._lock(repo.svfs, "repacklock", True, None,
                          None, _('repacking %s') % repo.origroot):
         pass
+
+def debugwaitonprefetch(repo):
+    with repo._lock(repo.svfs, "prefetchlock", True, None,
+                         None, _('prefetching in %s') % repo.origroot):
+        pass

remotefilelog/shallowrepo.py

@@ -5,10 +5,12 @@
 # This software may be used and distributed according to the terms of the
 # GNU General Public License version 2 or any later version.
 
+from hgext3rd.extutil import runshellcommand
 from mercurial.i18n import _
 from mercurial.node import hex, nullid, nullrev
 from mercurial import error, localrepo, util, match, scmutil
 from . import remotefilelog, remotefilectx, fileserverclient
+import repack as repackmod
 import constants, shallowutil
 from contentstore import remotefilelogcontentstore, unioncontentstore
 from contentstore import remotecontentstore
@@ -169,9 +171,36 @@ def wraprepo(repo):
             return super(shallowrepository, self).commitctx(ctx,
                                                             error=error)
 
-        def prefetch(self, revs, base=None, pats=None, opts=None):
-            """Prefetches all the necessary file revisions for the given revs
+        def backgroundprefetch(self, revs, base=None, repack=False, pats=None,
+                               opts=None):
+            """Runs prefetch in background with optional repack
             """
+            cmd = util.hgcmd() + ['-R', repo.origroot, 'prefetch']
+            if repack:
+                cmd.append('--repack')
+            if revs:
+                cmd += ['-r', revs]
+
+            cmd = ' '.join(map(util.shellquote, cmd))
+            msg = _("(running background prefetch)\n")
+
+            repo.ui.warn(msg)
+            runshellcommand(cmd, os.environ)
+
+        def prefetch(self, revs, base=None, repack=False, pats=None, opts=None):
+            """Prefetches all the necessary file revisions for the given revs
+            Optionally runs repack in background
+            """
+            with repo._lock(repo.svfs, 'prefetchlock', True, None, None,
+                            _('prefetching in %s') % repo.origroot):
+                self._prefetch(revs, base, repack, pats, opts)
+
+            # Run repack in background
+            if repack:
+                repackmod.backgroundrepack(repo, incremental=True)
+
+        def _prefetch(self, revs, base=None, repack=False, pats=None,
+                      opts=None):
             fallbackpath = self.fallbackpath
             if fallbackpath:
                 # If we know a rev is on the server, we should fetch the server

tests/test-remotefilelog-bgprefetch.t

@@ -0,0 +1,98 @@
+  $ PYTHONPATH=$TESTDIR/..:$PYTHONPATH
+  $ export PYTHONPATH
+
+  $ . "$TESTDIR/library.sh"
+
+  $ hginit master
+  $ cd master
+  $ cat >> .hg/hgrc <<EOF
+  > [remotefilelog]
+  > server=True
+  > EOF
+  $ echo x > x
+  $ echo z > z
+  $ hg commit -qAm x
+  $ echo x2 > x
+  $ echo y > y
+  $ hg commit -qAm y
+  $ hg bookmark foo
+
+  $ cd ..
+
+# prefetch a revision
+
+  $ hgcloneshallow ssh://user@dummy/master shallow --noupdate
+  streaming all changes
+  2 files to transfer, 528 bytes of data
+  transferred 528 bytes in * seconds (*/sec) (glob)
+  searching for changes
+  no changes found
+  $ cd shallow
+
+  $ hg prefetch -r 0
+  2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over *s (glob)
+
+  $ hg cat -r 0 x
+  x
+
+# background prefetch on pull when configured
+
+  $ cat >> .hg/hgrc <<EOF
+  > [remotefilelog]
+  > pullprefetch=bookmark()
+  > backgroundprefetch=True
+  > EOF
+  $ hg strip tip
+  saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/109c3a557a73-3f43405e-backup.hg (glob)
+
+  $ clearcache
+  $ hg pull
+  pulling from ssh://user@dummy/master
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 1 changesets with 0 changes to 0 files
+  updating bookmark foo
+  (run 'hg update' to get a working copy)
+  prefetching file contents
+  (running background prefetch)
+  $ sleep 0.5
+  $ hg debugwaitonprefetch >/dev/null 2>%1
+  $ find $CACHEDIR -type f | sort
+  $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/ef95c5376f34698742fe34f315fd82136f8f68c0
+  $TESTTMP/hgcache/master/39/5df8f7c51f007019cb30201c49e884b46b92fa/69a1b67522704ec122181c0890bd16e9d3e7516a
+  $TESTTMP/hgcache/master/95/cb0bfd2977c761298d9624e4b4d4c72a39974a/076f5e2225b3ff0400b98c92aa6cdf403ee24cca
+  $TESTTMP/hgcache/repos
+
+# background prefetch with repack on pull when configured
+
+  $ cat >> .hg/hgrc <<EOF
+  > [remotefilelog]
+  > backgroundrepack=True
+  > EOF
+  $ hg strip tip
+  saved backup bundle to $TESTTMP/shallow/.hg/strip-backup/109c3a557a73-3f43405e-backup.hg (glob)
+
+  $ clearcache
+  $ hg pull
+  pulling from ssh://user@dummy/master
+  searching for changes
+  adding changesets
+  adding manifests
+  adding file changes
+  added 1 changesets with 0 changes to 0 files
+  updating bookmark foo
+  (run 'hg update' to get a working copy)
+  prefetching file contents
+  (running background prefetch)
+  $ sleep 0.5
+  $ hg debugwaitonprefetch >/dev/null 2>%1
+  $ sleep 0.5
+  $ hg debugwaitonrepack >/dev/null 2>%1
+  $ find $CACHEDIR -type f | sort
+  $TESTTMP/hgcache/master/packs/47d8f1b90a73af4ff8af19fcd10bdc027b6a881a.histidx
+  $TESTTMP/hgcache/master/packs/47d8f1b90a73af4ff8af19fcd10bdc027b6a881a.histpack
+  $TESTTMP/hgcache/master/packs/830b4b13c2eeab214fdfe4635f27e3ad5e2dc730.dataidx
+  $TESTTMP/hgcache/master/packs/830b4b13c2eeab214fdfe4635f27e3ad5e2dc730.datapack
+  $TESTTMP/hgcache/repos