prefetch: merge prefetchtrees command into prefetch

Summary:
Currently,

 - `hg prefetch` prefetches files.
 - `hg prefetchtrees` prefetches trees.

This commit removes `prefetchtrees` and makes `prefetch` responsible for
everything i.e. `prefetch` will prefetch whatever it can prefetch be it files,
trees, or both.

Test Plan: Ran all the tests.

Reviewers: #fbhgext, durham

Reviewed By: #fbhgext, durham

Subscribers: quark, durham

Differential Revision: https://phab.mercurial-scm.org/D1417
This commit is contained in:
Saurabh Singh 2017-11-16 15:28:07 -08:00
parent fe26751847
commit 2520c63e45
2 changed files with 163 additions and 38 deletions

View File

@ -1,10 +1,27 @@
There are three cases which are of interest in this test:
- client remotefilelog enabled and the client repo is a shallowrepo
In this case, the expectation is that the prefetch command will be able to
prefetch both trees and files.
- client remotefilelog enabled and the client repo is not a shallowrepo
In this case, the expectation is that the prefetch command will be able to only
prefetch trees.
- client remotefilelog disabled
In this case, the expectation is that the prefetch command will be able to only
prefetch trees and will not be able to perform operations like repack which
require remotefilelog.
#testcases remotefilelog.true.shallowrepo.true remotefilelog.true.shallowrepo.false remotefilelog.false
$ CACHEDIR=`pwd`/hgcache
$ PYTHONPATH=$TESTDIR/..:$PYTHONPATH
$ export PYTHONPATH
$ . "$TESTDIR/library.sh"
$ hg init master
$ hginit master
$ cd master
$ mkdir dir
$ echo x > dir/x
@ -19,6 +36,7 @@
> treemanifest=$TESTDIR/../treemanifest
>
> [remotefilelog]
> server=True
> name=master
> cachepath=$CACHEDIR
> usefastdatapack=True
@ -32,6 +50,18 @@
> EOF
$ cd ..
#if remotefilelog.true.shallowrepo.true
$ hgcloneshallow ssh://user@dummy/master client
streaming all changes
2 files to transfer, 749 bytes of data
transferred 749 bytes in * seconds (*) (glob)
searching for changes
no changes found
updating to branch default
2 files updated, 0 files merged, 0 files removed, 0 files unresolved
2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob)
#else
$ hg clone ssh://user@dummy/master client
streaming all changes
4 files to transfer, 952 bytes of data
@ -40,6 +70,7 @@
no changes found
updating to branch default
2 files updated, 0 files merged, 0 files removed, 0 files unresolved
#endif
$ cd master
$ hg backfilltree
@ -57,15 +88,25 @@
> usecache = False
> EOF
Test prefetchtrees
$ hg prefetchtrees -r '0 + 1 + 2'
#if remotefilelog.false
$ cat >> .hg/hgrc <<EOF
>
> [extensions]
> remotefilelog=!
> EOF
#endif
Test prefetch
$ hg prefetch -r '0 + 1 + 2'
6 trees fetched over * (glob)
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !)
$ ls $CACHEDIR/master/packs/manifests
29938257d506f677320d5abec8e34a1a9ed635fe.histidx
29938257d506f677320d5abec8e34a1a9ed635fe.histpack
8adc618d23082c0a5311a4bbf9ac08b9b9672471.dataidx
8adc618d23082c0a5311a4bbf9ac08b9b9672471.datapack
$ hg debugdatapack --long $CACHEDIR/master/packs/manifests/*.dataidx
$ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \
> --long $CACHEDIR/master/packs/manifests/*.dataidx
$TESTTMP/hgcache/master/packs/manifests/8adc618d23082c0a5311a4bbf9ac08b9b9672471:
subdir:
Node Delta Base Delta Length Blob Size
@ -91,7 +132,8 @@ Test prefetchtrees
Node Delta Base Delta Length Blob Size
ef362f8bbe8aa457b0cfc49f200cbeb7747984ed 0000000000000000000000000000000000000000 46 (missing)
$ hg debughistorypack $CACHEDIR/master/packs/manifests/*.histidx
$ hg debughistorypack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \
> $CACHEDIR/master/packs/manifests/*.histidx
Node P1 Node P2 Node Link Node Copy From
@ -107,7 +149,8 @@ Test prefetchtrees
subdir
Node P1 Node P2 Node Link Node Copy From
ddb35f099a64 000000000000 000000000000 f15c65c6e9bd
$ hg debugdatapack --node ef362f8bbe8aa457b0cfc49f200cbeb7747984ed $CACHEDIR/master/packs/manifests/*.dataidx
$ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \
> --node ef362f8bbe8aa457b0cfc49f200cbeb7747984ed $CACHEDIR/master/packs/manifests/*.dataidx
$TESTTMP/hgcache/master/packs/manifests/8adc618d23082c0a5311a4bbf9ac08b9b9672471:
@ -126,12 +169,29 @@ Test prefetchtrees
Test prefetch with base node (subdir/ shouldn't show up in the pack)
$ rm -rf $CACHEDIR/master
$ hg prefetchtrees -r '2' --base '1'
#if remotefilelog.true.shallowrepo.true
Multiple trees are fetched in this case because the file prefetching code path
requires tree manifest for the base commit.
$ hg prefetch -r '2' --base '1'
2 trees fetched over * (glob)
2 trees fetched over * (glob)
3 trees fetched over * (glob)
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob)
$ ls $CACHEDIR/master/packs/manifests/*.dataidx
$TESTTMP/hgcache/master/packs/manifests/148e9eb32f473ea522c591c95be0f9e772be9675.dataidx
$TESTTMP/hgcache/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx
$TESTTMP/hgcache/master/packs/manifests/5f14647c5653622d4c2682648ec82c7193d2a9ab.dataidx
#else
$ hg prefetch -r '2' --base '1'
2 trees fetched over * (glob)
$ ls $CACHEDIR/master/packs/manifests/*.dataidx
$TESTTMP/hgcache/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx
#endif
$ hg debugdatapack $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx
$ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \
> $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx
$TESTTMP/hgcache/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c:
dir:
Node Delta Base Delta Length Blob Size
@ -160,6 +220,7 @@ Test auto prefetch during normal access
dir/x | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)
2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !)
$ ls $CACHEDIR/master/packs/manifests
148e9eb32f473ea522c591c95be0f9e772be9675.dataidx
148e9eb32f473ea522c591c95be0f9e772be9675.datapack
@ -170,7 +231,8 @@ Test auto prefetch during normal access
e5c44a5c1bbfd8841df1c6c4b7cca54536e016db.histidx
e5c44a5c1bbfd8841df1c6c4b7cca54536e016db.histpack
$ hg debugdatapack $CACHEDIR/master/packs/manifests/148e9eb32f473ea522c591c95be0f9e772be9675
$ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \
> $CACHEDIR/master/packs/manifests/148e9eb32f473ea522c591c95be0f9e772be9675
$TESTTMP/hgcache/master/packs/manifests/148e9eb32f473ea522c591c95be0f9e772be9675:
dir:
Node Delta Base Delta Length Blob Size
@ -186,7 +248,8 @@ Test auto prefetch during normal access
- Note that subdir/ is not downloaded again
$ hg debugdatapack $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c
$ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \
> $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c
$TESTTMP/hgcache/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c:
dir:
Node Delta Base Delta Length Blob Size
@ -199,14 +262,17 @@ Test auto prefetch during normal access
Test that auto prefetch scans up the changelog for base trees
$ rm -rf $CACHEDIR/master
$ hg prefetchtrees -r 'tip^'
$ hg prefetch -r 'tip^'
3 trees fetched over * (glob)
2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !)
$ rm -rf $CACHEDIR/master
$ hg prefetchtrees -r tip
$ hg prefetch -r tip
3 trees fetched over * (glob)
2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !)
- Only 2 of the 3 trees from tip^ are downloaded as part of --stat's fetch
$ hg log -r tip --stat --pager=off > /dev/null
2 trees fetched over * (glob)
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !)
Test auto prefetch during pull
@ -220,7 +286,8 @@ Test auto prefetch during pull
no changes found
prefetching trees
6 trees fetched over * (glob)
$ hg debugdatapack $CACHEDIR/master/packs/manifests/*.dataidx
$ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \
> $CACHEDIR/master/packs/manifests/*.dataidx
$TESTTMP/hgcache/master/packs/manifests/8adc618d23082c0a5311a4bbf9ac08b9b9672471:
subdir:
Node Delta Base Delta Length Blob Size
@ -257,7 +324,8 @@ Test auto prefetch during pull
no changes found
prefetching trees
3 trees fetched over * (glob)
$ hg debugdatapack $CACHEDIR/master/packs/manifests/*.dataidx
$ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \
> $CACHEDIR/master/packs/manifests/*.dataidx
$TESTTMP/hgcache/master/packs/manifests/4ee15de76c068ec1c80e3e61f2c3c476a779078a:
dir:
Node Delta Base Delta Length Blob Size
@ -274,8 +342,9 @@ Test auto prefetch during pull
- Prefetch commit 1 then minimally prefetch commit 2
$ rm -rf $CACHEDIR/master
$ hg prefetchtrees -r 1
$ hg prefetch -r 1
3 trees fetched over * (glob)
2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !)
$ ls $CACHEDIR/master/packs/manifests/*dataidx
$TESTTMP/hgcache/master/packs/manifests/148e9eb32f473ea522c591c95be0f9e772be9675.dataidx
$ hg pull --config treemanifest.pullprefetchcount=1 --traceback
@ -287,7 +356,8 @@ Test auto prefetch during pull
$ ls $CACHEDIR/master/packs/manifests/*dataidx
$TESTTMP/hgcache/master/packs/manifests/148e9eb32f473ea522c591c95be0f9e772be9675.dataidx
$TESTTMP/hgcache/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx
$ hg debugdatapack $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx
$ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \
> $CACHEDIR/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c.dataidx
$TESTTMP/hgcache/master/packs/manifests/3fb59713808147bda39cbd97b9cd862406f5865c:
dir:
Node Delta Base Delta Length Blob Size
@ -313,12 +383,14 @@ Test prefetching certain revs during pull
adding changesets
adding manifests
adding file changes
added 2 changesets with 2 changes to 1 files
added 2 changesets with 0 changes to 0 files (remotefilelog.true.shallowrepo.true !)
added 2 changesets with 2 changes to 1 files (no-remotefilelog.true.shallowrepo.true !)
new changesets dece825f8add:cfacdcc4cee5
(run 'hg update' to get a working copy)
prefetching trees
3 trees fetched over * (glob)
$ hg debugdatapack $CACHEDIR/master/packs/manifests/*.dataidx
$ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \
> $CACHEDIR/master/packs/manifests/*.dataidx
$TESTTMP/hgcache/master/packs/manifests/4ee15de76c068ec1c80e3e61f2c3c476a779078a:
dir:
Node Delta Base Delta Length Blob Size
@ -341,7 +413,8 @@ Test prefetching certain revs during pull
no changes found
prefetching trees
2 trees fetched over * (glob)
$ hg debugdatapack $CACHEDIR/master/packs/manifests/99050e724a9236121684523ba3f4db270e62fb58.dataidx
$ hg debugdatapack --config extensions.remotefilelog=$TESTDIR/../remotefilelog \
> $CACHEDIR/master/packs/manifests/99050e724a9236121684523ba3f4db270e62fb58.dataidx
$TESTTMP/hgcache/master/packs/manifests/99050e724a9236121684523ba3f4db270e62fb58:
dir:
Node Delta Base Delta Length Blob Size
@ -357,31 +430,43 @@ Test that prefetch refills just part of a tree when the cache is deleted
$ echo >> dir/x
$ hg commit -m 'edit x locally'
created new head
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !)
$ rm -rf $CACHEDIR/master/*
$ hg cat subdir/z
3 trees fetched over * (glob)
z
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !)
Test prefetch non-parent commits with no base node (should fetch minimal
trees - in this case 3 trees for commit 2, and 2 for commit 4 despite it having
3 directories)
$ rm -rf $CACHEDIR/master
$ hg prefetchtrees -r '2 + 4'
$ hg prefetch -r '2 + 4'
5 trees fetched over * (glob)
3 files fetched over 1 fetches - (3 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !)
Test repack option
$ rm -rf $CACHEDIR/master
$ hg prefetchtrees -r '0'
2 trees fetched over *s (glob)
$ hg prefetchtrees -r '2'
3 trees fetched over *s (glob)
$ hg prefetch -r '0'
2 trees fetched over * (glob)
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !)
$ hg prefetch -r '2'
3 trees fetched over * (glob)
2 files fetched over 1 fetches - (2 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !)
$ hg prefetchtrees -r '4' --repack
3 trees fetched over *s (glob)
#if remotefilelog.false
$ hg prefetch -r '4' --repack
abort: repack requires remotefilelog extension
[255]
#else
$ hg prefetch -r '4' --repack
3 trees fetched over * (glob)
(running background incremental repack)
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over * (glob) (remotefilelog.true.shallowrepo.true !)
$ sleep 0.5
$ hg debugwaitonrepack
$ ls_l $CACHEDIR/master/packs/manifests | grep datapack | wc -l
\s*1 (re)
#endif

View File

@ -82,6 +82,7 @@ from mercurial import (
registrar,
repair,
revlog,
scmutil,
sshserver,
templatekw,
util,
@ -90,6 +91,10 @@ from mercurial import (
from mercurial.i18n import _
from mercurial.node import bin, hex, nullid
from remotefilelog import (
cmdtable as remotefilelogcmdtable,
resolveprefetchopts,
)
from remotefilelog.contentstore import (
manifestrevlogstore,
unioncontentstore,
@ -177,6 +182,22 @@ def uisetup(ui):
# Change manifest template output
templatekw.defaulttempl['manifest'] = '{node}'
def _wrapremotefilelog(loaded):
if loaded:
remotefilelogmod = extensions.find('remotefilelog')
extensions.wrapcommand(
remotefilelogmod.cmdtable, 'prefetch', _prefetchwrapper)
else:
# There is no prefetch command to wrap around. In this case, we use
# the command table entry for prefetch in the remotefilelog to
# define the prefetch command, wrap it, and then override it
# completely. This ensures that the options to the prefetch command
# are consistent.
cmdtable['prefetch'] = remotefilelogcmdtable['prefetch']
extensions.wrapcommand(cmdtable, 'prefetch', _overrideprefetch)
extensions.afterloaded('remotefilelog', _wrapremotefilelog)
def showmanifest(orig, **args):
"""Same implementation as the upstream showmanifest, but without the 'rev'
field."""
@ -998,28 +1019,47 @@ def wrappropertycache(cls, propname, wrapper):
raise AttributeError(_("%s has no property '%s'") %
(type(currcls), propname))
@command('prefetchtrees', [
('r', 'rev', '', _("revs to prefetch the trees for")),
('', 'repack', False, _('run repack after prefetch')),
('b', 'base', '', _("rev that is assumed to already be local")),
] + commands.walkopts, _('--rev REVS PATTERN..'))
def prefetchtrees(ui, repo, *args, **opts):
revs = repo.revs(opts.get('rev'))
# Wrapper around the 'prefetch' command which also allows for prefetching the
# trees along with the files.
def _prefetchwrapper(orig, ui, repo, *pats, **opts):
# The wrapper will take care of the repacking.
repackrequested = opts.pop('repack')
_prefetchonlytrees(repo, opts)
_prefetchonlyfiles(orig, ui, repo, *pats, **opts)
if repackrequested:
backgroundrepack(repo, incremental=True)
# Wrapper around the 'prefetch' command which overrides the command completely
# and only allows for prefetching trees. This is only required when the
# 'prefetch' command is not available because the remotefilelog extension is not
# loaded and we want to be able to at least prefetch trees. The wrapping just
# ensures that we get a consistent interface to the 'prefetch' command.
def _overrideprefetch(orig, ui, repo, *pats, **opts):
if opts.get('repack'):
raise error.Abort(_('repack requires remotefilelog extension'))
_prefetchonlytrees(repo, opts)
def _prefetchonlyfiles(orig, ui, repo, *pats, **opts):
if shallowrepo.requirement in repo.requirements:
orig(ui, repo, *pats, **opts)
def _prefetchonlytrees(repo, opts):
opts = resolveprefetchopts(repo.ui, opts)
revs = scmutil.revrange(repo, opts.get('rev'))
mfnodes = set()
for rev in revs:
mfnodes.add(repo[rev].manifestnode())
basemfnode = set()
base = opts.get('base')
if base:
if base is not None:
basemfnode.add(repo[base].manifestnode())
_prefetchtrees(repo, '', mfnodes, basemfnode, [])
# Run repack in background
if opts.get('repack'):
backgroundrepack(repo, incremental=True)
def _prefetchtrees(repo, rootdir, mfnodes, basemfnodes, directories):
# If possible, use remotefilelog's more expressive fallbackpath
if util.safehasattr(repo, 'fallbackpath'):