2017-02-24 01:03:02 +03:00
|
|
|
#!/usr/bin/env python
|
2016-06-16 01:48:16 +03:00
|
|
|
import hashlib
|
2017-01-13 20:42:25 +03:00
|
|
|
import os
|
2016-04-29 01:00:34 +03:00
|
|
|
import random
|
|
|
|
import shutil
|
2017-01-13 20:42:25 +03:00
|
|
|
import stat
|
2016-04-29 01:00:34 +03:00
|
|
|
import struct
|
2017-01-13 20:42:25 +03:00
|
|
|
import sys
|
2016-04-29 01:00:34 +03:00
|
|
|
import tempfile
|
2016-05-03 22:32:16 +03:00
|
|
|
import time
|
2016-04-29 01:00:34 +03:00
|
|
|
import unittest
|
|
|
|
|
|
|
|
import silenttestrunner
|
|
|
|
|
2017-01-13 20:42:25 +03:00
|
|
|
# Load the local remotefilelog, not the system one
|
|
|
|
sys.path[0:0] = [os.path.join(os.path.dirname(__file__), '..')]
|
|
|
|
|
2016-05-16 20:59:09 +03:00
|
|
|
from remotefilelog.datapack import (
|
|
|
|
datapack,
|
2016-08-31 02:19:53 +03:00
|
|
|
fastdatapack,
|
2016-05-16 20:59:09 +03:00
|
|
|
mutabledatapack,
|
2016-05-20 19:31:37 +03:00
|
|
|
)
|
|
|
|
from remotefilelog.basepack import (
|
2016-05-16 20:59:09 +03:00
|
|
|
SMALLFANOUTCUTOFF,
|
|
|
|
SMALLFANOUTPREFIX,
|
|
|
|
LARGEFANOUTPREFIX,
|
|
|
|
)
|
2017-04-27 05:50:36 +03:00
|
|
|
from remotefilelog import constants
|
2016-04-29 01:00:34 +03:00
|
|
|
|
2016-11-25 03:23:21 +03:00
|
|
|
from mercurial.node import nullid
|
2016-09-28 21:53:29 +03:00
|
|
|
import mercurial.ui
|
2016-04-29 01:00:34 +03:00
|
|
|
|
2016-08-31 02:19:53 +03:00
|
|
|
class datapacktestsbase(object):
|
|
|
|
def __init__(self, datapackreader, paramsavailable):
|
|
|
|
self.datapackreader = datapackreader
|
|
|
|
self.paramsavailable = paramsavailable
|
|
|
|
|
2016-04-29 01:00:34 +03:00
|
|
|
def setUp(self):
|
|
|
|
self.tempdirs = []
|
|
|
|
|
|
|
|
def tearDown(self):
|
|
|
|
for d in self.tempdirs:
|
|
|
|
shutil.rmtree(d)
|
|
|
|
|
|
|
|
def makeTempDir(self):
|
|
|
|
tempdir = tempfile.mkdtemp()
|
|
|
|
self.tempdirs.append(tempdir)
|
|
|
|
return tempdir
|
|
|
|
|
|
|
|
def getHash(self, content):
|
2016-06-16 01:48:16 +03:00
|
|
|
return hashlib.sha1(content).digest()
|
2016-04-29 01:00:34 +03:00
|
|
|
|
|
|
|
def getFakeHash(self):
|
2016-05-16 20:59:09 +03:00
|
|
|
return ''.join(chr(random.randint(0, 255)) for _ in range(20))
|
2016-04-29 01:00:34 +03:00
|
|
|
|
2017-04-27 05:50:36 +03:00
|
|
|
def createPack(self, revisions=None, version=0):
|
2016-04-29 01:00:34 +03:00
|
|
|
if revisions is None:
|
|
|
|
revisions = [("filename", self.getFakeHash(), nullid, "content")]
|
|
|
|
|
|
|
|
packdir = self.makeTempDir()
|
2017-04-27 05:50:36 +03:00
|
|
|
packer = mutabledatapack(mercurial.ui.ui(), packdir, version=version)
|
2016-04-29 01:00:34 +03:00
|
|
|
|
2017-04-27 05:50:36 +03:00
|
|
|
for args in revisions:
|
|
|
|
filename, node, base, content = args[0:4]
|
|
|
|
# meta is optional
|
|
|
|
meta = None
|
|
|
|
if len(args) > 4:
|
|
|
|
meta = args[4]
|
|
|
|
packer.add(filename, node, base, content, metadata=meta)
|
2016-04-29 01:00:34 +03:00
|
|
|
|
|
|
|
path = packer.close()
|
2016-08-31 02:19:53 +03:00
|
|
|
return self.datapackreader(path)
|
2016-04-29 01:00:34 +03:00
|
|
|
|
|
|
|
def testAddSingle(self):
|
|
|
|
"""Test putting a simple blob into a pack and reading it out.
|
|
|
|
"""
|
|
|
|
filename = "foo"
|
|
|
|
content = "abcdef"
|
|
|
|
node = self.getHash(content)
|
|
|
|
|
|
|
|
revisions = [(filename, node, nullid, content)]
|
|
|
|
pack = self.createPack(revisions)
|
2016-08-31 02:19:53 +03:00
|
|
|
if self.paramsavailable:
|
|
|
|
self.assertEquals(pack.params.fanoutprefix, SMALLFANOUTPREFIX)
|
2016-04-29 01:00:34 +03:00
|
|
|
|
|
|
|
chain = pack.getdeltachain(filename, node)
|
|
|
|
self.assertEquals(content, chain[0][4])
|
|
|
|
|
|
|
|
def testAddMultiple(self):
|
|
|
|
"""Test putting multiple unrelated blobs into a pack and reading them
|
|
|
|
out.
|
|
|
|
"""
|
|
|
|
revisions = []
|
|
|
|
for i in range(10):
|
|
|
|
filename = "foo%s" % i
|
|
|
|
content = "abcdef%s" % i
|
|
|
|
node = self.getHash(content)
|
|
|
|
revisions.append((filename, node, nullid, content))
|
|
|
|
|
|
|
|
pack = self.createPack(revisions)
|
|
|
|
|
|
|
|
for filename, node, base, content in revisions:
|
|
|
|
chain = pack.getdeltachain(filename, node)
|
|
|
|
self.assertEquals(content, chain[0][4])
|
|
|
|
|
|
|
|
def testAddDeltas(self):
|
|
|
|
"""Test putting multiple delta blobs into a pack and read the chain.
|
|
|
|
"""
|
|
|
|
revisions = []
|
|
|
|
filename = "foo"
|
|
|
|
lastnode = nullid
|
|
|
|
for i in range(10):
|
|
|
|
content = "abcdef%s" % i
|
|
|
|
node = self.getHash(content)
|
|
|
|
revisions.append((filename, node, lastnode, content))
|
|
|
|
lastnode = node
|
|
|
|
|
|
|
|
pack = self.createPack(revisions)
|
|
|
|
# Test that the chain for the final entry has all the others
|
|
|
|
chain = pack.getdeltachain(filename, node)
|
|
|
|
for i in range(10):
|
|
|
|
content = "abcdef%s" % i
|
|
|
|
self.assertEquals(content, chain[-i - 1][4])
|
|
|
|
|
|
|
|
def testPackMany(self):
|
|
|
|
"""Pack many related and unrelated objects.
|
|
|
|
"""
|
|
|
|
# Build a random pack file
|
|
|
|
revisions = []
|
|
|
|
blobs = {}
|
|
|
|
random.seed(0)
|
|
|
|
for i in range(100):
|
|
|
|
filename = "filename-%s" % i
|
|
|
|
filerevs = []
|
|
|
|
for j in range(random.randint(1, 100)):
|
|
|
|
content = "content-%s" % j
|
|
|
|
node = self.getHash(content)
|
|
|
|
lastnode = nullid
|
|
|
|
if len(filerevs) > 0:
|
|
|
|
lastnode = filerevs[random.randint(0, len(filerevs) - 1)]
|
|
|
|
filerevs.append(node)
|
|
|
|
blobs[(filename, node, lastnode)] = content
|
|
|
|
revisions.append((filename, node, lastnode, content))
|
|
|
|
|
|
|
|
pack = self.createPack(revisions)
|
|
|
|
|
|
|
|
# Verify the pack contents
|
|
|
|
for (filename, node, lastnode), content in sorted(blobs.iteritems()):
|
|
|
|
chain = pack.getdeltachain(filename, node)
|
|
|
|
for entry in chain:
|
|
|
|
expectedcontent = blobs[(entry[0], entry[1], entry[3])]
|
|
|
|
self.assertEquals(entry[4], expectedcontent)
|
|
|
|
|
2017-04-27 05:50:36 +03:00
|
|
|
def testPackMetadata(self):
|
|
|
|
revisions = []
|
|
|
|
for i in range(10):
|
|
|
|
filename = '%s.txt' % i
|
|
|
|
content = ' \n' * i
|
|
|
|
node = self.getHash(content)
|
|
|
|
meta = {constants.METAKEYFLAG: i % 4,
|
2017-05-01 23:29:19 +03:00
|
|
|
constants.METAKEYSIZE: len(content),
|
|
|
|
'Z': 'random_string',
|
|
|
|
'_': '\0' * i}
|
2017-04-27 05:50:36 +03:00
|
|
|
revisions.append((filename, node, nullid, content, meta))
|
|
|
|
pack = self.createPack(revisions, version=1)
|
|
|
|
for name, node, x, content, origmeta in revisions:
|
|
|
|
parsedmeta = pack.getmeta(name, node)
|
|
|
|
# flag == 0 should be optimized out
|
|
|
|
if origmeta[constants.METAKEYFLAG] == 0:
|
|
|
|
del origmeta[constants.METAKEYFLAG]
|
|
|
|
self.assertEquals(parsedmeta, origmeta)
|
|
|
|
|
|
|
|
def testPackMetadataThrows(self):
|
|
|
|
filename = '1'
|
|
|
|
content = '2'
|
|
|
|
node = self.getHash(content)
|
|
|
|
meta = {constants.METAKEYFLAG: 3}
|
|
|
|
revisions = [(filename, node, nullid, content, meta)]
|
|
|
|
try:
|
|
|
|
self.createPack(revisions, version=0)
|
|
|
|
self.assertTrue(False, "should throw if metadata is not supported")
|
|
|
|
except RuntimeError:
|
|
|
|
pass
|
|
|
|
|
2016-04-29 01:00:34 +03:00
|
|
|
def testGetMissing(self):
|
|
|
|
"""Test the getmissing() api.
|
|
|
|
"""
|
|
|
|
revisions = []
|
|
|
|
filename = "foo"
|
|
|
|
lastnode = nullid
|
|
|
|
for i in range(10):
|
|
|
|
content = "abcdef%s" % i
|
|
|
|
node = self.getHash(content)
|
|
|
|
revisions.append((filename, node, lastnode, content))
|
|
|
|
lastnode = node
|
|
|
|
|
|
|
|
pack = self.createPack(revisions)
|
|
|
|
|
|
|
|
missing = pack.getmissing([("foo", revisions[0][1])])
|
|
|
|
self.assertFalse(missing)
|
|
|
|
|
|
|
|
missing = pack.getmissing([("foo", revisions[0][1]),
|
|
|
|
("foo", revisions[1][1])])
|
|
|
|
self.assertFalse(missing)
|
|
|
|
|
|
|
|
fakenode = self.getFakeHash()
|
|
|
|
missing = pack.getmissing([("foo", revisions[0][1]), ("foo", fakenode)])
|
|
|
|
self.assertEquals(missing, [("foo", fakenode)])
|
|
|
|
|
|
|
|
def testAddThrows(self):
|
|
|
|
pack = self.createPack()
|
|
|
|
|
|
|
|
try:
|
|
|
|
pack.add('filename', nullid, 'contents')
|
|
|
|
self.assertTrue(False, "datapack.add should throw")
|
|
|
|
except RuntimeError:
|
|
|
|
pass
|
|
|
|
|
|
|
|
def testBadVersionThrows(self):
|
|
|
|
pack = self.createPack()
|
|
|
|
path = pack.path + '.datapack'
|
|
|
|
with open(path) as f:
|
|
|
|
raw = f.read()
|
2017-04-27 05:50:36 +03:00
|
|
|
raw = struct.pack('!B', 255) + raw[1:]
|
2017-01-13 20:42:25 +03:00
|
|
|
os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
|
2016-04-29 01:00:34 +03:00
|
|
|
with open(path, 'w+') as f:
|
|
|
|
f.write(raw)
|
|
|
|
|
|
|
|
try:
|
2016-08-31 02:19:53 +03:00
|
|
|
pack = self.datapackreader(pack.path)
|
2016-04-29 01:00:34 +03:00
|
|
|
self.assertTrue(False, "bad version number should have thrown")
|
|
|
|
except RuntimeError:
|
|
|
|
pass
|
|
|
|
|
2016-05-16 20:59:09 +03:00
|
|
|
def testMissingDeltabase(self):
|
|
|
|
fakenode = self.getFakeHash()
|
|
|
|
revisions = [("filename", fakenode, self.getFakeHash(), "content")]
|
|
|
|
pack = self.createPack(revisions)
|
|
|
|
chain = pack.getdeltachain("filename", fakenode)
|
|
|
|
self.assertEquals(len(chain), 1)
|
|
|
|
|
2016-05-16 20:59:09 +03:00
|
|
|
def testLargePack(self):
|
|
|
|
"""Test creating and reading from a large pack with over X entries.
|
|
|
|
This causes it to use a 2^16 fanout table instead."""
|
|
|
|
revisions = []
|
|
|
|
blobs = {}
|
|
|
|
total = SMALLFANOUTCUTOFF + 1
|
|
|
|
for i in xrange(total):
|
|
|
|
filename = "filename-%s" % i
|
|
|
|
content = filename
|
|
|
|
node = self.getHash(content)
|
|
|
|
blobs[(filename, node)] = content
|
|
|
|
revisions.append((filename, node, nullid, content))
|
|
|
|
|
|
|
|
pack = self.createPack(revisions)
|
2016-08-31 02:19:53 +03:00
|
|
|
if self.paramsavailable:
|
|
|
|
self.assertEquals(pack.params.fanoutprefix, LARGEFANOUTPREFIX)
|
2016-05-16 20:59:09 +03:00
|
|
|
|
|
|
|
for (filename, node), content in blobs.iteritems():
|
|
|
|
actualcontent = pack.getdeltachain(filename, node)[0][4]
|
|
|
|
self.assertEquals(actualcontent, content)
|
|
|
|
|
2016-05-03 22:32:16 +03:00
|
|
|
# perf test off by default since it's slow
|
|
|
|
def _testIndexPerf(self):
|
|
|
|
random.seed(0)
|
|
|
|
print "Multi-get perf test"
|
|
|
|
packsizes = [
|
|
|
|
100,
|
|
|
|
10000,
|
|
|
|
100000,
|
|
|
|
500000,
|
|
|
|
1000000,
|
|
|
|
3000000,
|
|
|
|
]
|
|
|
|
lookupsizes = [
|
|
|
|
10,
|
|
|
|
100,
|
|
|
|
1000,
|
|
|
|
10000,
|
|
|
|
100000,
|
|
|
|
1000000,
|
|
|
|
]
|
|
|
|
for packsize in packsizes:
|
|
|
|
revisions = []
|
|
|
|
for i in xrange(packsize):
|
|
|
|
filename = "filename-%s" % i
|
|
|
|
content = "content-%s" % i
|
|
|
|
node = self.getHash(content)
|
|
|
|
revisions.append((filename, node, nullid, content))
|
|
|
|
|
|
|
|
path = self.createPack(revisions).path
|
|
|
|
|
|
|
|
# Perf of large multi-get
|
|
|
|
import gc
|
|
|
|
gc.disable()
|
2016-08-31 02:19:53 +03:00
|
|
|
pack = self.datapackreader(path)
|
2016-05-03 22:32:16 +03:00
|
|
|
for lookupsize in lookupsizes:
|
|
|
|
if lookupsize > packsize:
|
|
|
|
continue
|
|
|
|
random.shuffle(revisions)
|
|
|
|
findnodes = [(rev[0], rev[1]) for rev in revisions]
|
|
|
|
|
|
|
|
start = time.time()
|
2016-11-25 03:23:21 +03:00
|
|
|
pack.getmissing(findnodes[:lookupsize])
|
2016-05-03 22:32:16 +03:00
|
|
|
elapsed = time.time() - start
|
|
|
|
print ("%s pack %s lookups = %0.04f" %
|
|
|
|
(('%s' % packsize).rjust(7),
|
|
|
|
('%s' % lookupsize).rjust(7),
|
|
|
|
elapsed))
|
|
|
|
|
|
|
|
print ""
|
|
|
|
gc.enable()
|
|
|
|
|
|
|
|
# The perf test is meant to produce output, so we always fail the test
|
|
|
|
# so the user sees the output.
|
|
|
|
raise RuntimeError("perf test always fails")
|
|
|
|
|
2016-08-31 02:19:53 +03:00
|
|
|
class datapacktests(datapacktestsbase, unittest.TestCase):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
|
|
datapacktestsbase.__init__(self, datapack, True)
|
|
|
|
unittest.TestCase.__init__(self, *args, **kwargs)
|
|
|
|
|
|
|
|
class fastdatapacktests(datapacktestsbase, unittest.TestCase):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
|
|
datapacktestsbase.__init__(self, fastdatapack, False)
|
|
|
|
unittest.TestCase.__init__(self, *args, **kwargs)
|
|
|
|
|
2016-04-29 01:00:34 +03:00
|
|
|
# TODO:
|
|
|
|
# datapack store:
|
|
|
|
# - getmissing
|
|
|
|
# - GC two packs into one
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
silenttestrunner.main(__name__)
|