repack: add an option to run background repack with --packsonly

Summary:
Most of the repacks are background repacks, and most of the complaints are
coming from laptops users. Thanks to the rust repack, most of time during
repack is now spent in repacking loosefiles. While repacking them is expensive,
testing whether data is in a loosefile and obtaining it is actually pretty
fast. Packfiles have the opposite issue, repacking them is fast, but finding
data in them may be expensive if a lot of them are present.

Based on this, it makes sense to repack packfiles more frequently than
loosefiles. At first, the newly added option will be used to turn-off loosefile
repack for laptop users. A less frequent loosefile repack will be implemented
in a later patch.

Reviewed By: DurhamG

Differential Revision: D14586986

fbshipit-source-id: 5bc5c839cf8d2d78bcc4ffa016bbe3cf1b2ef3f7

edenscm/hgext/remotefilelog/__init__.py

@@ -85,6 +85,9 @@ Configs:
 
     ``remotefilelog.userustrepack`` use rust based repack implementation.
 
+    ``remotefilelog.packsonlyrepack`` only repack packfiles during background
+    repacks.
+
     ``remotefilelog.dolfsprefetch`` means that fileserverclient's prefetch
     will also cause lfs prefetch to happen. This is True by default.
 
@@ -1116,9 +1119,9 @@ def pull(orig, ui, repo, *pats, **opts):
             else:
                 repo.prefetch(revs, base=base)
                 if bgrepack:
-                    repackmod.backgroundrepack(repo, incremental=True)
+                    repackmod.domaintenancerepack(repo)
         elif bgrepack:
-            repackmod.backgroundrepack(repo, incremental=True)
+            repackmod.domaintenancerepack(repo)
 
     return result
 
@@ -1315,7 +1318,7 @@ def prefetch(ui, repo, *pats, **opts):
 
     # Run repack in background
     if opts.get("repack"):
-        repackmod.backgroundrepack(repo, incremental=True)
+        repackmod.domaintenancerepack(repo)
 
 
 @command(

edenscm/hgext/remotefilelog/fileserverclient.py

@@ -34,7 +34,7 @@ from . import constants, shallowutil, wirepack
 from .contentstore import unioncontentstore
 from .lz4wrapper import lz4decompress
 from .metadatastore import unionmetadatastore
-from .repack import backgroundrepack
+from .repack import domaintenancerepack
 
 
 # Statistics for debugging
@@ -190,7 +190,7 @@ class cacheconnection(object):
             # quickly, killing the performance of other commands. To avoid this
             # pathological case, we can run a quick repack on these files.
             if self.repo.ui.configbool("remotefilelog", "fetchpacks"):
-                backgroundrepack(self.repo, incremental=True, packsonly=True)
+                domaintenancerepack(self.repo)
 
             # The cacheclient may be executing expensive set commands in the
             # background, As soon as the exit command, or the pipe above is

edenscm/hgext/remotefilelog/repack.py

@@ -47,6 +47,17 @@ class RepackAlreadyRunning(error.Abort):
     pass
 
 
+def domaintenancerepack(repo):
+    """Perform a background repack if necessary.
+    """
+    packsonly = False
+
+    if repo.ui.configbool("remotefilelog", "packsonlyrepack"):
+        packsonly = True
+
+    backgroundrepack(repo, incremental=True, packsonly=packsonly, looseonly=False)
+
+
 def backgroundrepack(repo, incremental=True, packsonly=False, looseonly=False):
     cmd = [util.hgexecutable(), "-R", repo.origroot, "repack"]
     msg = _("(running background repack)\n")
@@ -54,8 +65,8 @@ def backgroundrepack(repo, incremental=True, packsonly=False, looseonly=False):
         cmd.append("--incremental")
         msg = _("(running background incremental repack)\n")
 
-    if looseonly and packsonly:
-        raise error.Abort("can't specify both looseonly and packsonly")
+    if not looseonly and repo.ui.configbool("remotefilelog", "packsonlyrepack"):
+        packsonly = True
 
     if packsonly:
         cmd.append("--packsonly")

edenscm/hgext/treemanifest/__init__.py

@@ -190,7 +190,7 @@ from ..remotefilelog.repack import (
     _computeincrementalhistorypack,
     _runrepack,
     _topacks,
-    backgroundrepack,
+    domaintenancerepack,
 )
 
 
@@ -1738,7 +1738,7 @@ def _prefetchwrapper(orig, ui, repo, *pats, **opts):
     _prefetchonlyfiles(orig, ui, repo, *pats, **opts)
 
     if repackrequested:
-        backgroundrepack(repo, incremental=True)
+        domaintenancerepack(repo)
 
 
 # Wrapper around the 'prefetch' command which overrides the command completely

tests/test-fb-hgext-remotefilelog-repack-packsonly.t

@@ -0,0 +1,99 @@
+
+  $ . "$TESTDIR/library.sh"
+
+  $ cat >> $HGRCPATH <<EOF
+  > [format]
+  > userustdatapack=True
+  > EOF
+
+  $ hginit master
+  $ cd master
+  $ cat >> .hg/hgrc <<EOF
+  > [remotefilelog]
+  > server=True
+  > serverexpiration=-1
+  > EOF
+  $ echo x > x
+  $ hg commit -qAm x
+  $ echo x >> x
+  $ hg commit -qAm x2
+  $ cd ..
+
+  $ hgcloneshallow ssh://user@dummy/master shallow -q
+  1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
+
+# Set the prefetchdays config to zero so that all commits are prefetched
+# no matter what their creation date is.
+  $ cd shallow
+  $ cat >> .hg/hgrc <<EOF
+  > [remotefilelog]
+  > prefetchdays=0
+  > userustrepack=True
+  > fetchpacks=True
+  > EOF
+  $ cd ..
+
+  $ cd shallow
+  $ find $CACHEDIR | sort
+  $TESTTMP/hgcache
+  $TESTTMP/hgcache/master
+  $TESTTMP/hgcache/master/11
+  $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072
+  $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/aee31534993a501858fb6dd96a065671922e7d51
+  $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/filename
+  $TESTTMP/hgcache/repos
+
+  $ cd ../master
+  $ echo x3 > x
+  $ hg commit -qAm x3
+  $ echo x4 > x
+  $ hg commit -qAm x4
+  $ cd ../shallow
+  $ hg pull -q
+  $ hg up -q tip
+  1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
+  $ hg prefetch -r 0
+  1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
+  $ hg prefetch -r 2
+  1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
+  $ find $CACHEDIR | sort
+  $TESTTMP/hgcache
+  $TESTTMP/hgcache/master
+  $TESTTMP/hgcache/master/11
+  $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072
+  $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/aee31534993a501858fb6dd96a065671922e7d51
+  $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/filename
+  $TESTTMP/hgcache/master/packs
+  $TESTTMP/hgcache/master/packs/1e6f0f575de6319f747ef83966a08775803fcecc.dataidx
+  $TESTTMP/hgcache/master/packs/1e6f0f575de6319f747ef83966a08775803fcecc.datapack
+  $TESTTMP/hgcache/master/packs/2d66e09c3bf8a000428af1630d978127182e496e.dataidx
+  $TESTTMP/hgcache/master/packs/2d66e09c3bf8a000428af1630d978127182e496e.datapack
+  $TESTTMP/hgcache/master/packs/3266aa7480df06153adccad2f1abb6d11f42de0e.dataidx
+  $TESTTMP/hgcache/master/packs/3266aa7480df06153adccad2f1abb6d11f42de0e.datapack
+  $TESTTMP/hgcache/master/packs/3b65e3071e408ff050835eba9d2662d0c5ea51db.histidx
+  $TESTTMP/hgcache/master/packs/3b65e3071e408ff050835eba9d2662d0c5ea51db.histpack
+  $TESTTMP/hgcache/master/packs/acb190832c13f0a23d7901bc1847ef7f6046a26e.histidx
+  $TESTTMP/hgcache/master/packs/acb190832c13f0a23d7901bc1847ef7f6046a26e.histpack
+  $TESTTMP/hgcache/master/packs/c3399b56e035f73c3295276ed098235a08a0ed8c.histidx
+  $TESTTMP/hgcache/master/packs/c3399b56e035f73c3295276ed098235a08a0ed8c.histpack
+  $TESTTMP/hgcache/repos
+
+  $ hg pull -q --config remotefilelog.backgroundrepack=True --config remotefilelog.packsonlyrepack=True
+  (running background incremental repack)
+  $ sleep 0.5
+  $ hg debugwaitonprefetch >/dev/null 2>%1
+
+  $ find $CACHEDIR | sort
+  $TESTTMP/hgcache
+  $TESTTMP/hgcache/master
+  $TESTTMP/hgcache/master/11
+  $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072
+  $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/aee31534993a501858fb6dd96a065671922e7d51
+  $TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/filename
+  $TESTTMP/hgcache/master/packs
+  $TESTTMP/hgcache/master/packs/3b65e3071e408ff050835eba9d2662d0c5ea51db.histidx
+  $TESTTMP/hgcache/master/packs/3b65e3071e408ff050835eba9d2662d0c5ea51db.histpack
+  $TESTTMP/hgcache/master/packs/591442b244487739875c540ec84be5a499a0b069.dataidx
+  $TESTTMP/hgcache/master/packs/591442b244487739875c540ec84be5a499a0b069.datapack
+  $TESTTMP/hgcache/master/packs/repacklock
+  $TESTTMP/hgcache/repos