2016-08-30 02:19:52 +03:00
|
|
|
# __init__.py
|
|
|
|
#
|
|
|
|
# Copyright 2016 Facebook, Inc.
|
|
|
|
#
|
|
|
|
# This software may be used and distributed according to the terms of the
|
|
|
|
# GNU General Public License version 2 or any later version.
|
|
|
|
|
2016-11-17 00:51:48 +03:00
|
|
|
"""allows using and migrating to tree manifests
|
|
|
|
|
|
|
|
When autocreatetrees is enabled, you can limit which bookmarks are initially
|
|
|
|
converted to trees during pull by specifying `treemanifest.allowedtreeroots`.
|
|
|
|
|
|
|
|
[treemanifest]
|
|
|
|
allowedtreeroots = master,stable
|
|
|
|
|
|
|
|
"""
|
|
|
|
|
2016-08-30 02:19:52 +03:00
|
|
|
from mercurial import (
|
2016-09-20 02:30:17 +03:00
|
|
|
changegroup,
|
2016-08-30 02:19:52 +03:00
|
|
|
cmdutil,
|
2016-12-03 01:37:51 +03:00
|
|
|
error,
|
2016-09-20 02:30:17 +03:00
|
|
|
extensions,
|
2016-08-30 02:19:52 +03:00
|
|
|
localrepo,
|
2016-09-20 02:30:17 +03:00
|
|
|
scmutil,
|
|
|
|
util,
|
2016-08-30 02:19:52 +03:00
|
|
|
)
|
2016-09-20 02:30:17 +03:00
|
|
|
from mercurial.i18n import _
|
2016-11-30 02:37:58 +03:00
|
|
|
from mercurial.node import bin, nullid
|
2016-09-20 02:30:17 +03:00
|
|
|
|
2016-08-30 02:19:52 +03:00
|
|
|
from remotefilelog.contentstore import unioncontentstore
|
2016-09-20 02:30:17 +03:00
|
|
|
from remotefilelog.datapack import datapackstore, mutabledatapack
|
2016-08-30 02:19:52 +03:00
|
|
|
from remotefilelog import shallowutil
|
|
|
|
import ctreemanifest
|
|
|
|
|
2016-09-20 02:30:17 +03:00
|
|
|
import struct
|
|
|
|
|
2016-08-30 02:19:52 +03:00
|
|
|
cmdtable = {}
|
|
|
|
command = cmdutil.command(cmdtable)
|
|
|
|
|
2016-10-21 21:02:22 +03:00
|
|
|
PACK_CATEGORY='manifests'
|
2016-08-30 02:19:52 +03:00
|
|
|
|
2016-09-20 02:30:17 +03:00
|
|
|
def extsetup(ui):
|
|
|
|
extensions.wrapfunction(changegroup.cg1unpacker, '_unpackmanifests',
|
|
|
|
_unpackmanifests)
|
|
|
|
|
2016-08-30 02:19:52 +03:00
|
|
|
def reposetup(ui, repo):
|
|
|
|
wraprepo(repo)
|
|
|
|
|
|
|
|
def wraprepo(repo):
|
|
|
|
if not isinstance(repo, localrepo.localrepository):
|
|
|
|
return
|
|
|
|
|
2016-12-03 01:37:51 +03:00
|
|
|
repo.name = repo.ui.config('remotefilelog', 'reponame')
|
|
|
|
if not repo.name:
|
|
|
|
raise error.Abort(_("remotefilelog.reponame must be configured"))
|
|
|
|
|
2016-10-21 21:02:22 +03:00
|
|
|
usecdatapack = repo.ui.configbool('remotefilelog', 'fastdatapack')
|
|
|
|
|
2016-10-21 21:02:20 +03:00
|
|
|
packpath = shallowutil.getcachepackpath(repo, PACK_CATEGORY)
|
2016-10-21 21:02:22 +03:00
|
|
|
datastore = datapackstore(packpath, usecdatapack=usecdatapack)
|
|
|
|
|
|
|
|
localpackpath = shallowutil.getlocalpackpath(repo.svfs.vfs.base,
|
|
|
|
PACK_CATEGORY)
|
|
|
|
localdatastore = datapackstore(localpackpath, usecdatapack=usecdatapack)
|
|
|
|
|
2016-12-03 01:37:45 +03:00
|
|
|
repo.svfs.sharedmanifestdatastores = [datastore]
|
|
|
|
repo.svfs.localmanifestdatastores = [localdatastore]
|
2016-10-21 21:02:22 +03:00
|
|
|
repo.svfs.manifestdatastore = unioncontentstore(localdatastore, datastore,
|
|
|
|
writestore=localdatastore)
|
2016-09-20 02:30:17 +03:00
|
|
|
|
|
|
|
def _unpackmanifests(orig, self, repo, *args, **kwargs):
|
2016-11-16 23:11:15 +03:00
|
|
|
mfrevlog = repo.manifestlog._revlog
|
|
|
|
oldtip = len(mfrevlog)
|
2016-09-20 02:30:17 +03:00
|
|
|
|
|
|
|
orig(self, repo, *args, **kwargs)
|
|
|
|
|
|
|
|
if (util.safehasattr(repo.svfs, "manifestdatastore") and
|
|
|
|
repo.ui.configbool('treemanifest', 'autocreatetrees')):
|
2016-10-21 21:02:20 +03:00
|
|
|
|
|
|
|
# TODO: only put in cache if pulling from main server
|
|
|
|
packpath = shallowutil.getcachepackpath(repo, PACK_CATEGORY)
|
2016-09-20 02:30:17 +03:00
|
|
|
opener = scmutil.vfs(packpath)
|
2016-09-21 23:51:39 +03:00
|
|
|
with mutabledatapack(repo.ui, opener) as dpack:
|
2016-11-16 23:11:15 +03:00
|
|
|
recordmanifest(dpack, repo, oldtip, len(mfrevlog))
|
2016-09-20 02:30:17 +03:00
|
|
|
dpack.close()
|
|
|
|
|
|
|
|
# Alert the store that there may be new packs
|
|
|
|
repo.svfs.manifestdatastore.markforrefresh()
|
|
|
|
|
|
|
|
class InterceptedMutablePack(object):
|
2016-10-21 21:02:26 +03:00
|
|
|
"""This classes intercepts pack writes and replaces the node for the root
|
|
|
|
with the provided node. This is useful for forcing a tree manifest to be
|
|
|
|
referencable via its flat hash.
|
|
|
|
"""
|
2016-11-30 02:37:58 +03:00
|
|
|
def __init__(self, pack, node, p1node):
|
2016-09-20 02:30:17 +03:00
|
|
|
self._pack = pack
|
|
|
|
self._node = node
|
2016-11-30 02:37:58 +03:00
|
|
|
self._p1node = p1node
|
2016-09-20 02:30:17 +03:00
|
|
|
|
|
|
|
def add(self, name, node, deltabasenode, delta):
|
|
|
|
# For the root node, provide the flat manifest as the key
|
|
|
|
if name == "":
|
|
|
|
node = self._node
|
2016-11-30 02:37:58 +03:00
|
|
|
if deltabasenode != nullid:
|
|
|
|
deltabasenode = self._p1node
|
2016-09-20 02:30:17 +03:00
|
|
|
return self._pack.add(name, node, deltabasenode, delta)
|
|
|
|
|
|
|
|
def recordmanifest(pack, repo, oldtip, newtip):
|
2016-11-16 23:11:15 +03:00
|
|
|
mfl = repo.manifestlog
|
|
|
|
mfrevlog = mfl._revlog
|
2016-09-20 02:30:17 +03:00
|
|
|
total = newtip - oldtip
|
|
|
|
ui = repo.ui
|
|
|
|
builttrees = {}
|
|
|
|
message = _('priming tree cache')
|
|
|
|
ui.progress(message, 0, total=total)
|
|
|
|
|
2016-09-21 23:51:39 +03:00
|
|
|
refcount = {}
|
|
|
|
for rev in xrange(oldtip, newtip):
|
2016-11-16 23:11:15 +03:00
|
|
|
p1 = mfrevlog.parentrevs(rev)[0]
|
|
|
|
p1node = mfrevlog.node(p1)
|
2016-09-21 23:51:39 +03:00
|
|
|
refcount[p1node] = refcount.get(p1node, 0) + 1
|
|
|
|
|
2016-11-17 00:51:48 +03:00
|
|
|
allowedtreeroots = set()
|
|
|
|
for name in repo.ui.configlist('treemanifest', 'allowedtreeroots'):
|
|
|
|
if name in repo:
|
|
|
|
allowedtreeroots.add(repo[name].manifestnode())
|
|
|
|
|
2016-09-20 02:30:17 +03:00
|
|
|
for rev in xrange(oldtip, newtip):
|
|
|
|
ui.progress(message, rev - oldtip, total=total)
|
2017-01-01 05:22:38 +03:00
|
|
|
p1, p2 = mfrevlog.parentrevs(rev)
|
2016-11-16 23:11:15 +03:00
|
|
|
p1node = mfrevlog.node(p1)
|
2017-01-01 05:22:38 +03:00
|
|
|
p2node = mfrevlog.node(p2)
|
2016-09-20 02:30:17 +03:00
|
|
|
|
2016-12-03 01:37:55 +03:00
|
|
|
if p1node == nullid:
|
|
|
|
origtree = ctreemanifest.treemanifest(repo.svfs.manifestdatastore)
|
|
|
|
elif p1node in builttrees:
|
2016-09-20 02:30:17 +03:00
|
|
|
origtree = builttrees[p1node]
|
|
|
|
else:
|
2016-11-16 23:11:15 +03:00
|
|
|
origtree = mfl[p1node].read()._treemanifest()
|
2016-09-20 02:30:17 +03:00
|
|
|
|
2016-11-30 02:37:58 +03:00
|
|
|
if origtree is None:
|
2016-11-17 00:51:48 +03:00
|
|
|
if allowedtreeroots and p1node not in allowedtreeroots:
|
|
|
|
continue
|
|
|
|
|
2016-11-16 23:11:15 +03:00
|
|
|
p1mf = mfl[p1node].read()
|
2016-09-20 02:30:17 +03:00
|
|
|
origtree = ctreemanifest.treemanifest(repo.svfs.manifestdatastore)
|
|
|
|
for filename, node, flag in p1mf.iterentries():
|
|
|
|
origtree.set(filename, node, flag)
|
2016-11-30 02:37:58 +03:00
|
|
|
origtree.write(InterceptedMutablePack(pack, p1node, nullid))
|
2016-09-20 02:30:17 +03:00
|
|
|
builttrees[p1node] = origtree
|
|
|
|
|
2016-09-21 23:51:39 +03:00
|
|
|
# Remove the tree from the cache once we've processed its final use.
|
|
|
|
# Otherwise memory explodes
|
|
|
|
p1refcount = refcount[p1node] - 1
|
|
|
|
if p1refcount == 0:
|
|
|
|
builttrees.pop(p1node, None)
|
|
|
|
refcount[p1node] = p1refcount
|
|
|
|
|
2017-01-01 05:22:38 +03:00
|
|
|
if p2node != nullid:
|
|
|
|
node = mfrevlog.node(rev)
|
|
|
|
diff = mfl[p1node].read().diff(mfl[node].read())
|
|
|
|
deletes = []
|
|
|
|
adds = []
|
|
|
|
for filename, ((anode, aflag), (bnode, bflag)) in diff.iteritems():
|
|
|
|
if bnode is None:
|
|
|
|
deletes.append(filename)
|
|
|
|
else:
|
|
|
|
adds.append((filename, bnode, bflag))
|
|
|
|
else:
|
|
|
|
# This will generally be very quick, since p1 == deltabase
|
|
|
|
delta = mfrevlog.revdiff(p1, rev)
|
|
|
|
|
|
|
|
deletes = []
|
|
|
|
adds = []
|
|
|
|
|
|
|
|
# Inspect the delta and read the added files from it
|
|
|
|
current = 0
|
|
|
|
end = len(delta)
|
|
|
|
while current < end:
|
|
|
|
try:
|
|
|
|
block = ''
|
|
|
|
# Deltas are of the form:
|
|
|
|
# <start><end><datalen><data>
|
|
|
|
# Where start and end say what bytes to delete, and data says
|
|
|
|
# what bytes to insert in their place. So we can just read
|
|
|
|
# <data> to figure out all the added files.
|
|
|
|
byte1, byte2, blocklen = struct.unpack(">lll",
|
|
|
|
delta[current:current + 12])
|
|
|
|
current += 12
|
|
|
|
if blocklen:
|
|
|
|
block = delta[current:current + blocklen]
|
|
|
|
current += blocklen
|
|
|
|
except struct.error:
|
|
|
|
raise RuntimeError("patch cannot be decoded")
|
|
|
|
|
|
|
|
# An individual delta block may contain multiple newline delimited
|
|
|
|
# entries.
|
|
|
|
for line in block.split('\n'):
|
|
|
|
if not line:
|
|
|
|
continue
|
|
|
|
fname, rest = line.split('\0')
|
|
|
|
fnode = rest[:40]
|
|
|
|
fflag = rest[40:]
|
|
|
|
adds.append((fname, bin(fnode), fflag))
|
|
|
|
|
|
|
|
allfiles = set(repo.changelog.readfiles(mfrevlog.linkrev(rev)))
|
|
|
|
deletes = allfiles.difference(fname for fname, fnode, fflag in adds)
|
2016-09-20 02:30:17 +03:00
|
|
|
|
|
|
|
# Apply the changes on top of the parent tree
|
|
|
|
newtree = origtree.copy()
|
|
|
|
for fname in deletes:
|
|
|
|
newtree.set(fname, None, None)
|
|
|
|
|
|
|
|
for fname, fnode, fflags in adds:
|
|
|
|
newtree.set(fname, fnode, fflags)
|
|
|
|
|
2016-11-30 02:37:58 +03:00
|
|
|
newtree.write(InterceptedMutablePack(pack, mfrevlog.node(rev), p1node),
|
2016-12-03 01:37:55 +03:00
|
|
|
origtree if p1node != nullid else None)
|
2016-09-20 02:30:17 +03:00
|
|
|
|
2016-09-21 23:57:06 +03:00
|
|
|
if ui.configbool('treemanifest', 'verifyautocreate', True):
|
2016-11-17 00:51:48 +03:00
|
|
|
diff = newtree.diff(origtree)
|
2016-09-21 23:57:06 +03:00
|
|
|
if len(diff) != len(adds) + len(deletes):
|
2016-09-20 02:30:17 +03:00
|
|
|
import pdb
|
|
|
|
pdb.set_trace()
|
|
|
|
|
2016-09-21 23:57:06 +03:00
|
|
|
for fname in deletes:
|
|
|
|
l, r = diff[fname]
|
|
|
|
if l != (None, ''):
|
|
|
|
import pdb
|
|
|
|
pdb.set_trace()
|
|
|
|
pass
|
|
|
|
|
|
|
|
for fname, fnode, fflags in adds:
|
|
|
|
l, r = diff[fname]
|
|
|
|
if l != (fnode, fflags):
|
|
|
|
import pdb
|
|
|
|
pdb.set_trace()
|
|
|
|
pass
|
2016-11-16 23:11:15 +03:00
|
|
|
builttrees[mfrevlog.node(rev)] = newtree
|
2016-09-20 02:30:17 +03:00
|
|
|
|
2016-11-16 23:11:15 +03:00
|
|
|
mfnode = mfrevlog.node(rev)
|
2016-09-21 23:51:39 +03:00
|
|
|
if refcount.get(mfnode) > 0:
|
|
|
|
builttrees[mfnode] = newtree
|
|
|
|
|
2016-09-20 02:30:17 +03:00
|
|
|
ui.progress(message, None)
|