mirror of
https://github.com/facebook/sapling.git
synced 2024-10-11 09:17:30 +03:00
111d46f0eb
Summary: This is a mod of perftest.py, where we remove the unneeded components and distill it into a simple correctness test. This is exposing bugs in our code which I will fix in subsequent diffs. Test Plan: well, it runs. it doesn't produce clean output yet though. :) Reviewers: #fastmanifest, durham Reviewed By: durham Subscribers: durham, mitrandir, mjpieters Differential Revision: https://phabricator.intern.facebook.com/D3866674 Signature: t1:3866674:1474225197:d147b18548e9e62c3b54e1ab1506141f3d52a07f
265 lines
8.4 KiB
Python
265 lines
8.4 KiB
Python
# treemanifest_correctness.py - simple extension for testing treemanifest
|
|
# correctness
|
|
#
|
|
# Copyright 2016 Facebook, Inc.
|
|
#
|
|
# This software may be used and distributed according to the terms of the
|
|
# GNU General Public License version 2 or any later version.
|
|
from mercurial import cmdutil, error, manifest, scmutil
|
|
from mercurial.node import nullid
|
|
from remotefilelog import datapack, contentstore, shallowutil
|
|
import difflib, pdb, hashlib, os, time
|
|
from fastmanifest.implementation import fastmanifestcache
|
|
from fastmanifest import cachemanager
|
|
import ctreemanifest
|
|
|
|
cmdtable = {}
|
|
command = cmdutil.command(cmdtable)
|
|
testedwith = ''
|
|
|
|
@command('testtree', [
|
|
('', 'build', '', ''),
|
|
('', 'revs', 'master + master~5', ''),
|
|
], '')
|
|
def testpackedtrees(ui, repo, *args, **opts):
|
|
packpath = shallowutil.getpackpath(repo, 'manifest')
|
|
if not os.path.exists(packpath):
|
|
os.mkdir(packpath)
|
|
opener = scmutil.vfs(packpath)
|
|
if opts.get('build'):
|
|
with datapack.mutabledatapack(opener) as newpack:
|
|
buildtreepack(repo, newpack, opts.get('build'))
|
|
newpack.close()
|
|
|
|
packstore = datapack.datapackstore(opener.base,
|
|
usecdatapack=ui.configbool('remotefilelog', 'fastdatapack'))
|
|
unionstore = contentstore.unioncontentstore(packstore)
|
|
|
|
ctxs = list(repo.set(opts.get('revs')))
|
|
profiletreepack(repo, unionstore, ctxs[0].hex(), ctxs[1].hex(), opts)
|
|
|
|
class cachestore(object):
|
|
def __init__(self):
|
|
self._cache = {}
|
|
|
|
def get(self, name, node):
|
|
return self._cache[(name, node)]
|
|
|
|
def getdeltachain(self, name, node):
|
|
data = self._cache[(name, node)]
|
|
return [(name, node, name, nullid, data)]
|
|
|
|
def add(self, name, node, data):
|
|
self._cache[(name, node)] = data
|
|
|
|
def buildtreepack(repo, pack, revs):
|
|
mf = repo.manifest
|
|
cache = cachestore()
|
|
packstore = datapack.datapackstore(pack.opener.base)
|
|
store = contentstore.unioncontentstore(cache, packstore)
|
|
ctxs = list(repo.set(revs))
|
|
for count, ctx in enumerate(ctxs):
|
|
repo.ui.progress(('manifests'), count, total=len(ctxs))
|
|
mfnode = ctx.manifestnode()
|
|
try:
|
|
store.get('', mfnode)
|
|
continue
|
|
except KeyError:
|
|
pass
|
|
|
|
try:
|
|
# if we have the parent tree, let's use it
|
|
p1, p2 = mf.parents(mfnode)
|
|
p1rev = mf.rev(p1)
|
|
mfrev = mf.rev(mfnode)
|
|
store.get('', p1)
|
|
|
|
if p2 == nullid and mf.deltaparent(mfrev) == p1rev:
|
|
mfdelta = mf.readdelta(mfnode)
|
|
adds = list((filename, n, f)
|
|
for (filename, n, f) in mfdelta.iterentries())
|
|
deletes = set(ctx.files()).difference(
|
|
filename
|
|
for filename, n, f in adds)
|
|
else:
|
|
mfctx = ctx.manifest()
|
|
mfdiff = mf.read(p1).diff(mfctx)
|
|
adds = list((f, bn, bf) for f, ((an, af), (bn, bf)) in
|
|
mfdiff.iteritems() if bn is not None)
|
|
deletes = list(f for f, ((an, af), (bn, bf)) in
|
|
mfdiff.iteritems() if bn is None)
|
|
|
|
tmfctx = read(store, '', p1).copy()
|
|
for filename in deletes:
|
|
del tmfctx[filename]
|
|
for filename, n, f in adds:
|
|
tmfctx[filename] = n
|
|
tmfctx.setflag(filename, f)
|
|
except KeyError:
|
|
mfctx = ctx.manifest()
|
|
tmfctx = manifest.treemanifest(text=mfctx.text())
|
|
|
|
p1, p2 = mf.parents(mfnode)
|
|
add(store, cache, pack, tmfctx, ctx.rev(), p1, p2,
|
|
forcenode=ctx.manifestnode())
|
|
repo.ui.progress(('manifests'), None)
|
|
|
|
def add(store, cache, pack, mf, linkrev, p1, p2, forcenode=False):
|
|
try:
|
|
store.get(mf._dir, p1)
|
|
p1mf = read(store, mf._dir, p1)
|
|
except KeyError:
|
|
p1mf = manifest.treemanifest()
|
|
|
|
try:
|
|
store.get(mf._dir, p2)
|
|
p2mf = read(store, mf._dir, p2)
|
|
except KeyError:
|
|
p2mf = manifest.treemanifest()
|
|
|
|
return _addtree(store, cache, pack, mf, linkrev,
|
|
p1mf, p2mf, forcenode=forcenode)
|
|
|
|
def read(store, dir, node):
|
|
def gettext():
|
|
return store.get(dir, node)
|
|
def readsubtree(dir, subm):
|
|
return read(store, dir, subm)
|
|
m = manifest.treemanifest(dir=dir)
|
|
m.read(gettext, readsubtree)
|
|
m.setnode(node)
|
|
return m
|
|
|
|
def _addtree(store, cache, pack, m, linkrev, m1, m2, forcenode=False):
|
|
# If the manifest is unchanged compared to one parent,
|
|
# don't write a new revision
|
|
if m.unmodifiedsince(m1) or m.unmodifiedsince(m2):
|
|
return m.node()
|
|
def writesubtree(subm, subp1, subp2):
|
|
add(store, cache, pack, subm, linkrev, subp1, subp2)
|
|
|
|
usemfv2 = False
|
|
m1._load()
|
|
m2._load()
|
|
m.writesubtrees(m1, m2, writesubtree)
|
|
text = m.dirtext(usemfv2)
|
|
# Double-check whether contents are unchanged to one parent
|
|
if text == m1.dirtext(usemfv2):
|
|
n = m1.node()
|
|
elif text == m2.dirtext(usemfv2):
|
|
n = m2.node()
|
|
else:
|
|
n = hashlib.sha1(m1.node() + m2.node() + text).digest()
|
|
# Save nodeid so parent manifest can calculate its nodeid
|
|
if forcenode:
|
|
n = forcenode
|
|
try:
|
|
store.get(m._dir, n)
|
|
except KeyError:
|
|
deltabase = nullid
|
|
delta = text
|
|
if False and m1.node() != nullid:
|
|
deltabase = m1.node()
|
|
delta = mdiff.textdiff(m1.dirtext(usemfv2), text)
|
|
pack.add(m._dir, n, deltabase, delta)
|
|
cache.add(m._dir, n, text)
|
|
m.setnode(n)
|
|
return n
|
|
|
|
def profiletreepack(repo, store, rev1, rev2, opts):
|
|
def exectest(name, prep, func):
|
|
elapsed = 0
|
|
elapsedprep = 0
|
|
args = []
|
|
if prep:
|
|
startprep = time.time()
|
|
args = prep()
|
|
elapsedprep += time.time() - startprep
|
|
import gc
|
|
gc.disable()
|
|
start = time.time()
|
|
result = func(*args)
|
|
elapsed += time.time() - start
|
|
gc.enable()
|
|
gc.collect()
|
|
repo.ui.progress(name, None)
|
|
|
|
total = elapsed + elapsedprep
|
|
repo.ui.status(("%0.2f" % (elapsedprep,)).ljust(15))
|
|
repo.ui.status(("%0.2f" % (elapsed,)).ljust(15))
|
|
repo.ui.status(("%0.2f" % (total,)).ljust(15))
|
|
|
|
return result
|
|
|
|
ctx1 = list(repo.set(rev1))[0]
|
|
ctx2 = list(repo.set(rev2))[0]
|
|
|
|
cachemanager.cachemanifestfillandtrim(
|
|
repo.ui, repo, ['%s + %s' % (ctx1.rev(), ctx2.rev())])
|
|
|
|
def flatconstructor(mfnode):
|
|
repo.manifest.clearcaches()
|
|
return repo.manifest.read(mfnode)._flatmanifest()
|
|
|
|
def ctreeconstructor(mfnode):
|
|
treemf = ctreemanifest.treemanifest(store, mfnode)
|
|
return treemf
|
|
|
|
# Test bodies
|
|
def prepone(new):
|
|
return [new(ctx1.manifestnode())]
|
|
def preptwo(new):
|
|
m1 = new(ctx1.manifestnode())
|
|
m2 = new(ctx2.manifestnode())
|
|
return m1, m2
|
|
|
|
def diff(m1, m2):
|
|
diff = m1.diff(m2)
|
|
|
|
result = []
|
|
for fp in sorted(diff.keys()):
|
|
result.append((fp, diff[fp]))
|
|
return result
|
|
def fulliter(m1):
|
|
entries = [x for x in m1]
|
|
return entries
|
|
|
|
tests = {
|
|
'diff': (preptwo, diff),
|
|
'fulliter': (prepone, fulliter),
|
|
}
|
|
|
|
kinds = {
|
|
'flat': flatconstructor,
|
|
'ctree': ctreeconstructor,
|
|
}
|
|
|
|
for testname, testopts in tests.items():
|
|
prepfunc, func = testopts
|
|
teststr = ('%s' % (testname,)).ljust(14)
|
|
repo.ui.status(("\n%sPrep Run Total\n") %
|
|
(teststr))
|
|
|
|
results = {}
|
|
for kind, kindconstructor in kinds.items():
|
|
repo.ui.status(("%s" % (kind)).ljust(14))
|
|
def prep():
|
|
return prepfunc(kindconstructor)
|
|
results[kind] = exectest(testname, prep, func)
|
|
repo.ui.status("\n")
|
|
repo.ui.status("\n")
|
|
|
|
correct = results['flat']
|
|
uut = results['ctree']
|
|
|
|
s = difflib.SequenceMatcher(None, uut, correct)
|
|
|
|
for tag, i1, i2, j1, j2 in s.get_opcodes():
|
|
if tag != 'equal':
|
|
repo.ui.status(("%7s a[%d:%d] (%s) b[%d:%d] (%s)\n" %
|
|
(tag, i1, i2, uut[i1:i2],
|
|
j1, j2, correct[j1:j2])))
|
|
else:
|
|
repo.ui.status(("%7s a[%d:%d] b[%d:%d]\n" %
|
|
(tag, i1, i2, j1, j2)))
|