mirror of
https://github.com/facebook/sapling.git
synced 2024-10-08 07:49:11 +03:00
remotefilelog: use python's repack logic for loosefiles
Summary: Loosefiles are still being fetched from memcache, and thus until we switch memcache to provide pack files, repack needs to be able to handle loosefiles. We could either implement the loosefile repack logic in rust, or we could just let the python code do it. Since we plan on having memcache serve pack files soon, the second approach is prefered as it result in the least amount of work. Reviewed By: DurhamG Differential Revision: D13707886 fbshipit-source-id: 5373b61f9749c158e6dfb026f6b12a7dd2a67802
This commit is contained in:
Xavier Deguillard
Facebook Github Bot
parent
245c21b8ee
commit
1538e82829
@ -620,44 +620,15 @@ class repacker(object):
|
||||
self.keepkeys = keepset(repo, lambda f, n: (f, n))
|
||||
self.isold = isold
|
||||
|
||||
def _userustrepack(self):
|
||||
return self.repo.ui.configbool("remotefilelog", "userustrepack", False)
|
||||
|
||||
def run(self, packpath, targetdata, targethistory):
|
||||
ledger = repackledger()
|
||||
|
||||
with flock(
|
||||
repacklockvfs(self.repo).join("repacklock"),
|
||||
_("repacking %s") % self.repo.origroot,
|
||||
timeout=0,
|
||||
):
|
||||
self.repo.hook("prerepack")
|
||||
|
||||
_cleanuptemppacks(self.repo.ui, self.packpath)
|
||||
|
||||
if self._userustrepack():
|
||||
try:
|
||||
if self.history == self.fullhistory:
|
||||
# full repack
|
||||
repackdatapacks(packpath, targetdata.opener.base)
|
||||
repackhistpacks(packpath, targethistory.opener.base)
|
||||
else:
|
||||
repackincrementaldatapacks(packpath, targetdata.opener.base)
|
||||
repackincrementalhistpacks(packpath, targethistory.opener.base)
|
||||
return
|
||||
except Exception as e:
|
||||
self.repo.ui.warn(
|
||||
_("warning: rust repack failed, fallback to python: %s\n") % e
|
||||
)
|
||||
|
||||
def _runpythonrepack(self, ledger, packpath, targetdata, targethistory, options):
|
||||
# Populate ledger from source
|
||||
with progress.spinner(
|
||||
self.repo.ui,
|
||||
_("scanning for %s %s to repack") % (self.sharedstr, self.unit),
|
||||
) as prog:
|
||||
ledger.prog = prog
|
||||
self.data.markledger(ledger, options=self.options)
|
||||
self.history.markledger(ledger, options=self.options)
|
||||
self.data.markledger(ledger, options=options)
|
||||
self.history.markledger(ledger, options=options)
|
||||
ledger.prog = None
|
||||
|
||||
# Run repack
|
||||
@ -676,6 +647,53 @@ class repacker(object):
|
||||
for cleanup in ledger.cleanup:
|
||||
cleanup(self.repo.ui)
|
||||
|
||||
def _runrustrepack(self, ledger, packpath, targetdata, targethistory):
|
||||
if not (self.options and self.options.get(constants.OPTION_LOOSEONLY)):
|
||||
try:
|
||||
if not (self.options and self.options.get(constants.OPTION_PACKSONLY)):
|
||||
options = dict(self.options)
|
||||
options[constants.OPTION_LOOSEONLY] = True
|
||||
self._runpythonrepack(
|
||||
ledger, packpath, targetdata, targethistory, options
|
||||
)
|
||||
|
||||
if self.history == self.fullhistory:
|
||||
# full repack
|
||||
repackdatapacks(packpath, targetdata.opener.base)
|
||||
repackhistpacks(packpath, targethistory.opener.base)
|
||||
else:
|
||||
repackincrementaldatapacks(packpath, targetdata.opener.base)
|
||||
repackincrementalhistpacks(packpath, targethistory.opener.base)
|
||||
return
|
||||
except Exception as e:
|
||||
self.repo.ui.warn(
|
||||
_("warning: rust repack failed, fallback to python: %s\n") % e
|
||||
)
|
||||
|
||||
self._runpythonrepack(ledger, packpath, targetdata, targethistory, self.options)
|
||||
|
||||
def _userustrepack(self):
|
||||
return self.repo.ui.configbool("remotefilelog", "userustrepack", False)
|
||||
|
||||
def run(self, packpath, targetdata, targethistory):
|
||||
ledger = repackledger()
|
||||
|
||||
with flock(
|
||||
repacklockvfs(self.repo).join("repacklock"),
|
||||
_("repacking %s") % self.repo.origroot,
|
||||
timeout=0,
|
||||
):
|
||||
self.repo.hook("prerepack")
|
||||
|
||||
_cleanuptemppacks(self.repo.ui, self.packpath)
|
||||
|
||||
if self._userustrepack():
|
||||
self._runrustrepack(ledger, packpath, targetdata, targethistory)
|
||||
else:
|
||||
self._runpythonrepack(
|
||||
ledger, packpath, targetdata, targethistory, self.options
|
||||
)
|
||||
|
||||
def _chainorphans(self, ui, filename, nodes, orphans, deltabases):
|
||||
"""Reorderes ``orphans`` into a single chain inside ``nodes`` and
|
||||
``deltabases``.
|
||||
|
||||
@ -4,8 +4,6 @@
|
||||
$ cat >> $HGRCPATH <<EOF
|
||||
> [format]
|
||||
> userustdatapack=True
|
||||
> [remotefilelog]
|
||||
> fetchpacks=True
|
||||
> EOF
|
||||
|
||||
$ hginit master
|
||||
@ -22,7 +20,7 @@
|
||||
$ cd ..
|
||||
|
||||
$ hgcloneshallow ssh://user@dummy/master shallow -q
|
||||
1 files fetched over 1 fetches - (0 misses, 100.00% hit ratio) over *s (glob)
|
||||
1 files fetched over 1 fetches - (1 misses, 0.00% hit ratio) over *s (glob)
|
||||
|
||||
# Set the prefetchdays config to zero so that all commits are prefetched
|
||||
# no matter what their creation date is.
|
||||
@ -31,33 +29,19 @@
|
||||
> [remotefilelog]
|
||||
> prefetchdays=0
|
||||
> userustrepack=True
|
||||
> fetchpacks=True
|
||||
> EOF
|
||||
$ cd ..
|
||||
|
||||
# Test that repack cleans up the old files and creates new packs
|
||||
|
||||
$ cd shallow
|
||||
$ find $CACHEDIR | sort
|
||||
$TESTTMP/hgcache
|
||||
$TESTTMP/hgcache/master
|
||||
$TESTTMP/hgcache/master/packs
|
||||
$TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histidx
|
||||
$TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histpack
|
||||
$TESTTMP/hgcache/master/packs/887690f1138ae5b99c50d754ed02262874bf8ecb.dataidx
|
||||
$TESTTMP/hgcache/master/packs/887690f1138ae5b99c50d754ed02262874bf8ecb.datapack
|
||||
|
||||
$ hg repack
|
||||
|
||||
# Repacking one datapack/historypack should result in the same datapack/historypack
|
||||
$ find $CACHEDIR | sort
|
||||
$TESTTMP/hgcache
|
||||
$TESTTMP/hgcache/master
|
||||
$TESTTMP/hgcache/master/packs
|
||||
$TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histidx
|
||||
$TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histpack
|
||||
$TESTTMP/hgcache/master/packs/887690f1138ae5b99c50d754ed02262874bf8ecb.dataidx
|
||||
$TESTTMP/hgcache/master/packs/887690f1138ae5b99c50d754ed02262874bf8ecb.datapack
|
||||
$TESTTMP/hgcache/master/packs/repacklock
|
||||
$TESTTMP/hgcache/master/11
|
||||
$TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072
|
||||
$TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/aee31534993a501858fb6dd96a065671922e7d51
|
||||
$TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/filename
|
||||
$TESTTMP/hgcache/repos
|
||||
|
||||
$ cd ../master
|
||||
$ echo x3 > x
|
||||
@ -69,7 +53,7 @@
|
||||
$ hg up -q tip
|
||||
1 files fetched over 1 fetches - (0 misses, 100.00% hit ratio) over 0.00s
|
||||
|
||||
# Pack multiple datapack/historypack into one
|
||||
# Pack a mix of packfiles and loosefiles into one packfile
|
||||
$ hg prefetch -r 0
|
||||
1 files fetched over 1 fetches - (0 misses, 100.00% hit ratio) over * (glob)
|
||||
$ hg prefetch -r 2
|
||||
@ -77,31 +61,45 @@
|
||||
$ find $CACHEDIR | sort
|
||||
$TESTTMP/hgcache
|
||||
$TESTTMP/hgcache/master
|
||||
$TESTTMP/hgcache/master/11
|
||||
$TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072
|
||||
$TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/aee31534993a501858fb6dd96a065671922e7d51
|
||||
$TESTTMP/hgcache/master/11/f6ad8ec52a2984abaafd7c3b516503785c2072/filename
|
||||
$TESTTMP/hgcache/master/packs
|
||||
$TESTTMP/hgcache/master/packs/1e6f0f575de6319f747ef83966a08775803fcecc.dataidx
|
||||
$TESTTMP/hgcache/master/packs/1e6f0f575de6319f747ef83966a08775803fcecc.datapack
|
||||
$TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histidx
|
||||
$TESTTMP/hgcache/master/packs/276d308429d0303762befa376788300f0310f90e.histpack
|
||||
$TESTTMP/hgcache/master/packs/2d66e09c3bf8a000428af1630d978127182e496e.dataidx
|
||||
$TESTTMP/hgcache/master/packs/2d66e09c3bf8a000428af1630d978127182e496e.datapack
|
||||
$TESTTMP/hgcache/master/packs/3266aa7480df06153adccad2f1abb6d11f42de0e.dataidx
|
||||
$TESTTMP/hgcache/master/packs/3266aa7480df06153adccad2f1abb6d11f42de0e.datapack
|
||||
$TESTTMP/hgcache/master/packs/3b65e3071e408ff050835eba9d2662d0c5ea51db.histidx
|
||||
$TESTTMP/hgcache/master/packs/3b65e3071e408ff050835eba9d2662d0c5ea51db.histpack
|
||||
$TESTTMP/hgcache/master/packs/887690f1138ae5b99c50d754ed02262874bf8ecb.dataidx
|
||||
$TESTTMP/hgcache/master/packs/887690f1138ae5b99c50d754ed02262874bf8ecb.datapack
|
||||
$TESTTMP/hgcache/master/packs/acb190832c13f0a23d7901bc1847ef7f6046a26e.histidx
|
||||
$TESTTMP/hgcache/master/packs/acb190832c13f0a23d7901bc1847ef7f6046a26e.histpack
|
||||
$TESTTMP/hgcache/master/packs/c3399b56e035f73c3295276ed098235a08a0ed8c.histidx
|
||||
$TESTTMP/hgcache/master/packs/c3399b56e035f73c3295276ed098235a08a0ed8c.histpack
|
||||
$TESTTMP/hgcache/master/packs/repacklock
|
||||
$TESTTMP/hgcache/repos
|
||||
|
||||
$ hg repack
|
||||
$ ls_l $TESTTMP/hgcache/master/packs/ | grep datapack
|
||||
-r-------- 253 073bc5bae3cee0940d7f1983cab3fe6754ed1407.datapack
|
||||
-r-------- 257 bd97e4d840a67a68dc7b1851615edf4b53c18bd4.datapack
|
||||
$ ls_l $TESTTMP/hgcache/master/packs/ | grep histpack
|
||||
-r-------- 336 3b65e3071e408ff050835eba9d2662d0c5ea51db.histpack
|
||||
|
||||
$ hg repack
|
||||
|
||||
# Repacking one datapack/historypack should result in the same datapack/historypack
|
||||
$ find $CACHEDIR | sort
|
||||
$TESTTMP/hgcache
|
||||
$TESTTMP/hgcache/master
|
||||
$TESTTMP/hgcache/master/packs
|
||||
$TESTTMP/hgcache/master/packs/3b65e3071e408ff050835eba9d2662d0c5ea51db.histidx
|
||||
$TESTTMP/hgcache/master/packs/3b65e3071e408ff050835eba9d2662d0c5ea51db.histpack
|
||||
$TESTTMP/hgcache/master/packs/bd97e4d840a67a68dc7b1851615edf4b53c18bd4.dataidx
|
||||
$TESTTMP/hgcache/master/packs/bd97e4d840a67a68dc7b1851615edf4b53c18bd4.datapack
|
||||
$TESTTMP/hgcache/master/packs/repacklock
|
||||
$TESTTMP/hgcache/repos
|
||||
|
||||
$ hg cat -r . x
|
||||
x4
|
||||
$ hg cat -r '.^' x
|
||||
|
||||
Loading…
Reference in New Issue
Block a user