hg: add debuglfsdownloadsize command

Summary:
This can be convenient when the size of LFS downloads between r1 and r2 is needed.

NB: this does not pay attention to sparse profiles and acts as if the checkout was full. We probably need to know how to do both.

Reviewed By: quark-zju

Differential Revision: D6938259

fbshipit-source-id: 52ab88be83339472f2eccafc746a191ff26c16c7

hgext/lfs/__init__.py

@@ -40,6 +40,7 @@ from mercurial import (
     bundle2,
     changegroup,
     context,
+    error,
     exchange,
     extensions,
     filelog,
@@ -49,6 +50,7 @@ from mercurial import (
     revlog,
     scmutil,
     upgrade,
+    util,
     vfs as vfsmod,
 )
 
@@ -235,3 +237,56 @@ def debuglfsreceive(ui, oid, size, url=None):
     remote.readbatch(pointers, local)
 
     ui.write((local.read(oid)))
+
+@command('debuglfsdownload', [
+         ('r', 'rev', [], _('revision'), _('REV')),
+         ('', 'sparse', True, _('respect sparse profile, '
+                                        '(otherwise check all files)')),
+         ], _('hg debuglfsdownload -r REV1 -r REV2'), norepo=False)
+def debuglfsdownload(ui, repo, *pats, **opts):
+    """calculate the LFS download size when updating between REV1 and REV2
+
+    If --no-sparse is provided, this operation would ignore any sparse
+    profile that might be present and report data for the full checkout.
+
+    With -v also prints which files are to be downloaded and the size of
+    each file."""
+    revs = opts.get('rev')
+
+    node1, node2 = scmutil.revpair(repo, revs)
+    match = lambda s: True
+    if not opts.get('sparse'):
+        ui.debug('will ignore sparse profile in this repo\n')
+    else:
+        if not util.safehasattr(repo, 'sparsematch'):
+            raise error.Abort(_('--ignore-sparse makes no sense in a non-sparse'
+                                ' repository'))
+        match = repo.sparsematch(node2)
+
+    with ui.configoverride({('remotefilelog', 'dolfsprefetch'): False}):
+        ctx1, ctx2 = repo[node1], repo[node2]
+        mfdiff = ctx2.manifest().diff(ctx1.manifest())
+        lfsflogs = util.sortdict()  # LFS filelogs
+        for fname in mfdiff:
+            if not match(fname):
+                continue
+            flog = repo.file(fname)
+            try:
+                node = ctx2.filenode(fname)
+            except error.ManifestLookupError:
+                continue
+            if wrapper._islfs(flog, node=node):
+                lfsflogs[fname] = flog
+
+        totalsize = 0
+        presentsize = 0
+        for fname, flog in lfsflogs.items():
+            rawtext = flog.revision(ctx2.filenode(fname), raw=True)
+            p = pointer.deserialize(rawtext)
+            present = repo.svfs.lfslocalblobstore.has(p.oid())
+            lfssize = int(p['size'])
+            ui.note(_("%s: %i (present=%r)\n") % (fname, lfssize, present))
+            totalsize += lfssize
+            presentsize += lfssize if present else 0
+        ui.status(_("Total size: %i, to download: %i, already exists: %r\n") %
+                  (totalsize, totalsize - presentsize, presentsize))

hgext/remotefilelog/__init__.py

@@ -52,6 +52,8 @@ Configs:
     for it to be considered for an incremental history repack.
     ``remotefilelog.history.repacksizelimit`` the maximum total size of pack
     files to include in an incremental history repack.
+    ``remotefilelog.dolfsprefetch`` means that fileserverclient's prefetch
+    will also cause lfs prefetch to happen. This is True by default.
 """
 from __future__ import absolute_import
 
@@ -76,7 +78,6 @@ from mercurial import (
     cmdutil,
     commands,
     context,
-    context,
     copies,
     debugcommands as hgdebugcommands,
     dispatch,

hgext/remotefilelog/fileserverclient.py

@@ -618,7 +618,8 @@ class fileserverclient(object):
                 raise error.Abort(_("unable to download %d files") %
                                   len(missingids))
             fetchcost += time.time() - start
-            self._lfsprefetch(fileids)
+            if self.ui.configbool('remotefilelog', 'dolfsprefetch', True):
+                self._lfsprefetch(fileids)
 
     def _lfsprefetch(self, fileids):
         if not _lfsmod or not util.safehasattr(

tests/test-check-module-imports.t

@@ -152,13 +152,13 @@ outputs, which should be fixed later.
   hgext/pushrebase.py:50: imports from mercurial.node not lexically sorted: hex < nullid
   hgext/pushrebase.py:50: imports from mercurial.node not lexically sorted: bin < hex
   hgext/pushrebase.py:51: symbol import follows non-symbol import: mercurial.i18n
-  hgext/remotefilelog/__init__.py:70: symbol import follows non-symbol import: mercurial.node
-  hgext/remotefilelog/__init__.py:71: symbol import follows non-symbol import: mercurial.i18n
-  hgext/remotefilelog/__init__.py:72: direct symbol import wrapfunction from mercurial.extensions
-  hgext/remotefilelog/__init__.py:72: symbol import follows non-symbol import: mercurial.extensions
-  hgext/remotefilelog/__init__.py:102: stdlib import "os" follows local import: mercurial
-  hgext/remotefilelog/__init__.py:103: stdlib import "time" follows local import: mercurial
-  hgext/remotefilelog/__init__.py:104: stdlib import "traceback" follows local import: mercurial
+  hgext/remotefilelog/__init__.py:72: symbol import follows non-symbol import: mercurial.node
+  hgext/remotefilelog/__init__.py:73: symbol import follows non-symbol import: mercurial.i18n
+  hgext/remotefilelog/__init__.py:74: direct symbol import wrapfunction from mercurial.extensions
+  hgext/remotefilelog/__init__.py:74: symbol import follows non-symbol import: mercurial.extensions
+  hgext/remotefilelog/__init__.py:103: stdlib import "os" follows local import: mercurial
+  hgext/remotefilelog/__init__.py:104: stdlib import "time" follows local import: mercurial
+  hgext/remotefilelog/__init__.py:105: stdlib import "traceback" follows local import: mercurial
   hgext/remotefilelog/basestore.py:3: multiple imported names: errno, hashlib, os, shutil, stat, time
   hgext/remotefilelog/basestore.py:15: symbol import follows non-symbol import: mercurial.i18n
   hgext/remotefilelog/basestore.py:16: symbol import follows non-symbol import: mercurial.node