repack: delete excessively big packfiles before incremental repack

Abnormally huge packfiles (20 GB+) are unwanted and can cause more problems
than they solve. Let's add a config to simply delete them at the beginning of
the repack proces.

Differential Revision: https://phab.mercurial-scm.org/D1523

remotefilelog/__init__.py

@@ -11,6 +11,7 @@ only download them ondemand as needed.
 Configs:
 
     ``packs.maxchainlen`` specifies the maximum delta chain length in pack files
+    ``packs.maxpacksize`` specifies the maximum pack file size
     ``remotefilelog.backgroundprefetch`` runs prefetch in background when True
     ``remotefilelog.bgprefetchrevs`` specifies revisions to fetch on commit and
       update, and on other commands that use them. Different from pullprefetch.

remotefilelog/repack.py

@@ -150,12 +150,39 @@ def _topacks(packpath, files, constructor):
     packs = list(constructor(p) for p in paths)
     return packs
 
+def _deletebigpacks(repo, folder, files):
+    """Deletes packfiles that are bigger than ``packs.maxpacksize``.
+
+    Returns ``files` with the removed files omitted."""
+    maxsize = repo.ui.configbytes("packs", "maxpacksize")
+    if maxsize <= 0:
+        return files
+
+    # This only considers datapacks today, but we could broaden it to include
+    # historypacks.
+    VALIDEXTS = [".datapack", ".dataidx"]
+
+    # Either an oversize index or datapack will trigger cleanup of the whole
+    # pack:
+    oversized = set([os.path.splitext(path)[0] for path, ftype, stat in files
+        if (stat.st_size > maxsize and (os.path.splitext(path)[1]
+                                        in VALIDEXTS))])
+
+    for rootfname in oversized:
+        rootpath = os.path.join(folder, rootfname)
+        for ext in VALIDEXTS:
+            path = rootpath + ext
+            repo.ui.debug('removing oversize packfile %s (%s)\n' %
+                          (path, util.bytecount(os.stat(path).st_size)))
+            os.unlink(path)
+    return [row for row in files if os.path.basename(row[0]) not in oversized]
+
 def _incrementalrepack(repo, datastore, historystore, packpath, category,
         allowincompletedata=False, options=None):
     shallowutil.mkstickygroupdir(repo.ui, packpath)
 
     files = osutil.listdir(packpath, stat=True)
-
+    files = _deletebigpacks(repo, packpath, files)
     datapacks = _topacks(packpath,
         _computeincrementaldatapack(repo.ui, files),
         datapack.datapack)

tests/test-remotefilelog-repack.t

@@ -427,3 +427,9 @@ Test limiting the max delta chain length
   577959738234  000000000000  8             8
   
   Total:                      8             8         (0.0% bigger)
+
+Test huge pack cleanup using different values of packs.maxpacksize:
+  $ hg repack --incremental --debug
+  $ hg repack --incremental --debug --config packs.maxpacksize=512
+  removing oversize packfile $TESTTMP/hgcache/master/packs/a2731c9a16403457b67337a620931797fce8c821.datapack (365 bytes)
+  removing oversize packfile $TESTTMP/hgcache/master/packs/a2731c9a16403457b67337a620931797fce8c821.dataidx (1.21 KB)