mirror of
https://github.com/facebook/sapling.git
synced 2024-10-11 09:17:30 +03:00
d3c6def7b8
Summary: Treemanifest had a bug where the pack files it created were 400 instead of 444. This meant people sharing a cache on the same machine couldn't access them. In most of the remotefilelog code we had set opener.createmode before creating the pack but we forgot to for treemanifest. This patch moves the opener creation and createmode setting into the mutable pack class so we can't screw this up again. Test Plan: Tests that wrote to the packs before, now failed and had to be updated to chmod the packs before writing to them. Reviewers: #mercurial, rmcelroy Reviewed By: rmcelroy Subscribers: rmcelroy, mjpieters Differential Revision: https://phabricator.intern.facebook.com/D4411580 Tasks: 15469140 Signature: t1:4411580:1484321293:9aa78254677548a6dc2270c58cee0ec6f57dd089
267 lines
8.3 KiB
Python
267 lines
8.3 KiB
Python
# treemanifest_correctness.py - simple extension for testing treemanifest
|
|
# correctness
|
|
#
|
|
# Copyright 2016 Facebook, Inc.
|
|
#
|
|
# This software may be used and distributed according to the terms of the
|
|
# GNU General Public License version 2 or any later version.
|
|
from mercurial import (
|
|
cmdutil,
|
|
manifest,
|
|
mdiff,
|
|
)
|
|
from mercurial.node import nullid
|
|
from remotefilelog import datapack, contentstore, shallowutil
|
|
import difflib, hashlib, os, time
|
|
from fastmanifest import cachemanager
|
|
import ctreemanifest
|
|
|
|
cmdtable = {}
|
|
command = cmdutil.command(cmdtable)
|
|
testedwith = 'ships-with-fb-hgext'
|
|
|
|
@command('testtree', [
|
|
('', 'build', '', ''),
|
|
('', 'revs', 'master + master~5', ''),
|
|
], '')
|
|
def testpackedtrees(ui, repo, *args, **opts):
|
|
packpath = shallowutil.getcachepackpath(repo, 'manifest')
|
|
if not os.path.exists(packpath):
|
|
os.mkdir(packpath)
|
|
if opts.get('build'):
|
|
with datapack.mutabledatapack(ui, packpath) as newpack:
|
|
buildtreepack(repo, newpack, opts.get('build'))
|
|
newpack.close()
|
|
|
|
packstore = datapack.datapackstore(ui, packpath,
|
|
usecdatapack=ui.configbool('remotefilelog', 'fastdatapack'))
|
|
unionstore = contentstore.unioncontentstore(packstore)
|
|
|
|
ctxs = list(repo.set(opts.get('revs')))
|
|
profiletreepack(repo, unionstore, ctxs[0].hex(), ctxs[1].hex(), opts)
|
|
|
|
class cachestore(object):
|
|
def __init__(self):
|
|
self._cache = {}
|
|
|
|
def get(self, name, node):
|
|
return self._cache[(name, node)]
|
|
|
|
def getdeltachain(self, name, node):
|
|
data = self._cache[(name, node)]
|
|
return [(name, node, name, nullid, data)]
|
|
|
|
def add(self, name, node, data):
|
|
self._cache[(name, node)] = data
|
|
|
|
def buildtreepack(repo, pack, revs):
|
|
mf = repo.manifest
|
|
cache = cachestore()
|
|
packstore = datapack.datapackstore(repo.ui, pack.opener.base)
|
|
store = contentstore.unioncontentstore(cache, packstore)
|
|
ctxs = list(repo.set(revs))
|
|
for count, ctx in enumerate(ctxs):
|
|
repo.ui.progress(('manifests'), count, total=len(ctxs))
|
|
mfnode = ctx.manifestnode()
|
|
try:
|
|
store.get('', mfnode)
|
|
continue
|
|
except KeyError:
|
|
pass
|
|
|
|
try:
|
|
# if we have the parent tree, let's use it
|
|
p1, p2 = mf.parents(mfnode)
|
|
p1rev = mf.rev(p1)
|
|
mfrev = mf.rev(mfnode)
|
|
store.get('', p1)
|
|
|
|
if p2 == nullid and mf.deltaparent(mfrev) == p1rev:
|
|
mfdelta = mf.readdelta(mfnode)
|
|
adds = list((filename, n, f)
|
|
for (filename, n, f) in mfdelta.iterentries())
|
|
deletes = set(ctx.files()).difference(
|
|
filename
|
|
for filename, n, f in adds)
|
|
else:
|
|
mfctx = ctx.manifest()
|
|
mfdiff = mf.read(p1).diff(mfctx)
|
|
adds = list((f, bn, bf) for f, ((an, af), (bn, bf)) in
|
|
mfdiff.iteritems() if bn is not None)
|
|
deletes = list(f for f, ((an, af), (bn, bf)) in
|
|
mfdiff.iteritems() if bn is None)
|
|
|
|
tmfctx = read(store, '', p1).copy()
|
|
for filename in deletes:
|
|
del tmfctx[filename]
|
|
for filename, n, f in adds:
|
|
tmfctx[filename] = n
|
|
tmfctx.setflag(filename, f)
|
|
except KeyError:
|
|
mfctx = ctx.manifest()
|
|
tmfctx = manifest.treemanifest(text=mfctx.text())
|
|
|
|
p1, p2 = mf.parents(mfnode)
|
|
add(store, cache, pack, tmfctx, ctx.rev(), p1, p2,
|
|
forcenode=ctx.manifestnode())
|
|
repo.ui.progress(('manifests'), None)
|
|
|
|
def add(store, cache, pack, mf, linkrev, p1, p2, forcenode=False):
|
|
try:
|
|
store.get(mf._dir, p1)
|
|
p1mf = read(store, mf._dir, p1)
|
|
except KeyError:
|
|
p1mf = manifest.treemanifest()
|
|
|
|
try:
|
|
store.get(mf._dir, p2)
|
|
p2mf = read(store, mf._dir, p2)
|
|
except KeyError:
|
|
p2mf = manifest.treemanifest()
|
|
|
|
return _addtree(store, cache, pack, mf, linkrev,
|
|
p1mf, p2mf, forcenode=forcenode)
|
|
|
|
def read(store, dir, node):
|
|
def gettext():
|
|
return store.get(dir, node)
|
|
def readsubtree(dir, subm):
|
|
return read(store, dir, subm)
|
|
m = manifest.treemanifest(dir=dir)
|
|
m.read(gettext, readsubtree)
|
|
m.setnode(node)
|
|
return m
|
|
|
|
def _addtree(store, cache, pack, m, linkrev, m1, m2, forcenode=False):
|
|
# If the manifest is unchanged compared to one parent,
|
|
# don't write a new revision
|
|
if m.unmodifiedsince(m1) or m.unmodifiedsince(m2):
|
|
return m.node()
|
|
def writesubtree(subm, subp1, subp2):
|
|
add(store, cache, pack, subm, linkrev, subp1, subp2)
|
|
|
|
usemfv2 = False
|
|
m1._load()
|
|
m2._load()
|
|
m.writesubtrees(m1, m2, writesubtree)
|
|
text = m.dirtext(usemfv2)
|
|
# Double-check whether contents are unchanged to one parent
|
|
if text == m1.dirtext(usemfv2):
|
|
n = m1.node()
|
|
elif text == m2.dirtext(usemfv2):
|
|
n = m2.node()
|
|
else:
|
|
n = hashlib.sha1(m1.node() + m2.node() + text).digest()
|
|
# Save nodeid so parent manifest can calculate its nodeid
|
|
if forcenode:
|
|
n = forcenode
|
|
try:
|
|
store.get(m._dir, n)
|
|
except KeyError:
|
|
deltabase = nullid
|
|
delta = text
|
|
if False and m1.node() != nullid:
|
|
deltabase = m1.node()
|
|
delta = mdiff.textdiff(m1.dirtext(usemfv2), text)
|
|
pack.add(m._dir, n, deltabase, delta)
|
|
cache.add(m._dir, n, text)
|
|
m.setnode(n)
|
|
return n
|
|
|
|
def profiletreepack(repo, store, rev1, rev2, opts):
|
|
def exectest(name, prep, func):
|
|
elapsed = 0
|
|
elapsedprep = 0
|
|
args = []
|
|
if prep:
|
|
startprep = time.time()
|
|
args = prep()
|
|
elapsedprep += time.time() - startprep
|
|
import gc
|
|
gc.disable()
|
|
start = time.time()
|
|
result = func(*args)
|
|
elapsed += time.time() - start
|
|
gc.enable()
|
|
gc.collect()
|
|
repo.ui.progress(name, None)
|
|
|
|
total = elapsed + elapsedprep
|
|
repo.ui.status(("%0.2f" % (elapsedprep,)).ljust(15))
|
|
repo.ui.status(("%0.2f" % (elapsed,)).ljust(15))
|
|
repo.ui.status(("%0.2f" % (total,)).ljust(15))
|
|
|
|
return result
|
|
|
|
ctx1 = list(repo.set(rev1))[0]
|
|
ctx2 = list(repo.set(rev2))[0]
|
|
|
|
cachemanager.cachemanifestfillandtrim(
|
|
repo.ui, repo, ['%s + %s' % (ctx1.rev(), ctx2.rev())])
|
|
|
|
def flatconstructor(mfnode):
|
|
repo.manifest.clearcaches()
|
|
return repo.manifest.read(mfnode)._flatmanifest()
|
|
|
|
def ctreeconstructor(mfnode):
|
|
treemf = ctreemanifest.treemanifest(store, mfnode)
|
|
return treemf
|
|
|
|
# Test bodies
|
|
def prepone(new):
|
|
return [new(ctx1.manifestnode())]
|
|
def preptwo(new):
|
|
m1 = new(ctx1.manifestnode())
|
|
m2 = new(ctx2.manifestnode())
|
|
return m1, m2
|
|
|
|
def diff(m1, m2):
|
|
diff = m1.diff(m2)
|
|
|
|
result = []
|
|
for fp in sorted(diff.keys()):
|
|
result.append((fp, diff[fp]))
|
|
return result
|
|
def fulliter(m1):
|
|
entries = [x for x in m1]
|
|
return entries
|
|
|
|
tests = {
|
|
'diff': (preptwo, diff),
|
|
'fulliter': (prepone, fulliter),
|
|
}
|
|
|
|
kinds = {
|
|
'flat': flatconstructor,
|
|
'ctree': ctreeconstructor,
|
|
}
|
|
|
|
for testname, testopts in tests.items():
|
|
prepfunc, func = testopts
|
|
teststr = ('%s' % (testname,)).ljust(14)
|
|
repo.ui.status(("\n%sPrep Run Total\n") %
|
|
(teststr))
|
|
|
|
results = {}
|
|
for kind, kindconstructor in kinds.items():
|
|
repo.ui.status(("%s" % (kind)).ljust(14))
|
|
def prep():
|
|
return prepfunc(kindconstructor)
|
|
results[kind] = exectest(testname, prep, func)
|
|
repo.ui.status("\n")
|
|
repo.ui.status("\n")
|
|
|
|
correct = results['flat']
|
|
uut = results['ctree']
|
|
|
|
s = difflib.SequenceMatcher(None, uut, correct)
|
|
|
|
for tag, i1, i2, j1, j2 in s.get_opcodes():
|
|
if tag != 'equal':
|
|
repo.ui.status(("%7s a[%d:%d] (%s) b[%d:%d] (%s)\n" %
|
|
(tag, i1, i2, uut[i1:i2],
|
|
j1, j2, correct[j1:j2])))
|
|
else:
|
|
repo.ui.status(("%7s a[%d:%d] b[%d:%d]\n" %
|
|
(tag, i1, i2, j1, j2)))
|