mirror of
https://github.com/facebook/sapling.git
synced 2024-10-12 09:48:05 +03:00
90e4ab0d4e
Summary:
After D15266191, the bindings crate directly depends on pyrevisionstore, and
since then we have seen really weird compilation errors that magically go away
when some files are touched.
From my observations, here is what I came up with. The pyrevisionstore crate is
both compiled as a dependency of the bindings crate, and as a library to be
used in python. Therefore, its Cargo.toml contains a '[lib]' section, the
presence of this section forces the crate to be compiled into a
"libpyrevisionstore.{rlib,so}", while all the regular crates have a hashed
suffix like "libedenapi-2b9975ec9b865e87.rlib".
None of this would usually matter, but our build system re-uses the build
directory to then compile the pyrevisionstore library. While doing so, it will
re-create the "libpyrevisionstore.{rlib,so}", but not in an identical fashion.
After this, the rest of the build succeeds.
Once a file in the bindings crate is touched, recompiling will only recompile
its file, and not the pyrevisionstore crate, but since
"libpyrevisionstore.{rlib,so}" is different, it now fails to compile...
A previous effort tried to compile each top level target into a separate
directory, but that directly affected some of our tests. Since the underlying
issue is that pyrevisionstore is compiled twice, let's simply not compile it as
a top-level target and simply fold it into the bindings lib. Ideally, we would
want to do the opposite, but let's do that at a later time.
Reviewed By: DurhamG
Differential Revision: D15381038
fbshipit-source-id: 047cfab794371464942f19084ffb9240e836cb40
572 lines
19 KiB
Python
Executable File
572 lines
19 KiB
Python
Executable File
#!/usr/bin/env python
|
|
from __future__ import absolute_import, print_function
|
|
|
|
import hashlib
|
|
import os
|
|
import random
|
|
import shutil
|
|
import stat
|
|
import struct
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
|
|
import edenscm.mercurial.ui as uimod
|
|
import silenttestrunner
|
|
from edenscm.hgext.remotefilelog import constants
|
|
from edenscm.hgext.remotefilelog.basepack import (
|
|
LARGEFANOUTPREFIX,
|
|
SMALLFANOUTCUTOFF,
|
|
SMALLFANOUTPREFIX,
|
|
)
|
|
from edenscm.hgext.remotefilelog.datapack import (
|
|
datapack,
|
|
datapackstore,
|
|
fastdatapack,
|
|
mutabledatapack,
|
|
)
|
|
from edenscm.hgext.remotefilelog.pyrevisionstore import datapack as rustdatapack
|
|
from edenscm.mercurial.node import nullid
|
|
|
|
|
|
try:
|
|
xrange(0)
|
|
except NameError:
|
|
xrange = range
|
|
|
|
|
|
class datapacktestsbase(object):
|
|
def __init__(self, datapackreader, paramsavailable, iscdatapack):
|
|
self.datapackreader = datapackreader
|
|
self.iscdatapack = iscdatapack
|
|
self.paramsavailable = paramsavailable
|
|
|
|
def setUp(self):
|
|
self.tempdirs = []
|
|
|
|
def tearDown(self):
|
|
for d in self.tempdirs:
|
|
shutil.rmtree(d)
|
|
|
|
def makeTempDir(self):
|
|
tempdir = tempfile.mkdtemp()
|
|
self.tempdirs.append(tempdir)
|
|
return tempdir
|
|
|
|
def getHash(self, content):
|
|
return hashlib.sha1(content).digest()
|
|
|
|
def getFakeHash(self):
|
|
return "".join(chr(random.randint(0, 255)) for _ in range(20))
|
|
|
|
def createPack(self, revisions=None, packdir=None, version=0):
|
|
if revisions is None:
|
|
revisions = [("filename", self.getFakeHash(), nullid, "content")]
|
|
|
|
if packdir is None:
|
|
packdir = self.makeTempDir()
|
|
|
|
packer = mutabledatapack(uimod.ui(), packdir, version=version)
|
|
|
|
for args in revisions:
|
|
filename, node, base, content = args[0:4]
|
|
# meta is optional
|
|
meta = None
|
|
if len(args) > 4:
|
|
meta = args[4]
|
|
packer.add(filename, node, base, content, metadata=meta)
|
|
|
|
path = packer.close()
|
|
return self.datapackreader(path)
|
|
|
|
def _testAddSingle(self, content):
|
|
"""Test putting a simple blob into a pack and reading it out.
|
|
"""
|
|
filename = "foo"
|
|
node = self.getHash(content)
|
|
|
|
revisions = [(filename, node, nullid, content)]
|
|
pack = self.createPack(revisions)
|
|
if self.paramsavailable:
|
|
self.assertEquals(pack.params.fanoutprefix, SMALLFANOUTPREFIX)
|
|
|
|
chain = pack.getdeltachain(filename, node)
|
|
self.assertEquals(content, chain[0][4])
|
|
|
|
def testAddSingle(self):
|
|
self._testAddSingle("")
|
|
|
|
def testAddSingleEmpty(self):
|
|
self._testAddSingle("abcdef")
|
|
|
|
def testAddMultiple(self):
|
|
"""Test putting multiple unrelated blobs into a pack and reading them
|
|
out.
|
|
"""
|
|
revisions = []
|
|
for i in range(10):
|
|
filename = "foo%s" % i
|
|
content = "abcdef%s" % i
|
|
node = self.getHash(content)
|
|
revisions.append((filename, node, self.getFakeHash(), content))
|
|
|
|
pack = self.createPack(revisions)
|
|
|
|
for filename, node, base, content in revisions:
|
|
entry = pack.getdelta(filename, node)
|
|
self.assertEquals((content, filename, base, {}), entry)
|
|
|
|
chain = pack.getdeltachain(filename, node)
|
|
self.assertEquals(content, chain[0][4])
|
|
|
|
def testAddDeltas(self):
|
|
"""Test putting multiple delta blobs into a pack and read the chain.
|
|
"""
|
|
revisions = []
|
|
filename = "foo"
|
|
lastnode = nullid
|
|
for i in range(10):
|
|
content = "abcdef%s" % i
|
|
node = self.getHash(content)
|
|
revisions.append((filename, node, lastnode, content))
|
|
lastnode = node
|
|
|
|
pack = self.createPack(revisions)
|
|
|
|
entry = pack.getdelta(filename, revisions[0][1])
|
|
realvalue = (revisions[0][3], filename, revisions[0][2], {})
|
|
self.assertEquals(entry, realvalue)
|
|
|
|
# Test that the chain for the final entry has all the others
|
|
chain = pack.getdeltachain(filename, node)
|
|
for i in range(10):
|
|
content = "abcdef%s" % i
|
|
self.assertEquals(content, chain[-i - 1][4])
|
|
|
|
def testPackMany(self):
|
|
"""Pack many related and unrelated objects.
|
|
"""
|
|
# Build a random pack file
|
|
revisions = []
|
|
blobs = {}
|
|
random.seed(0)
|
|
for i in range(100):
|
|
filename = "filename-%s" % i
|
|
filerevs = []
|
|
for j in range(random.randint(1, 100)):
|
|
content = "content-%s-%s" % (i, j)
|
|
node = self.getHash(content)
|
|
lastnode = nullid
|
|
if len(filerevs) > 0:
|
|
lastnode = filerevs[random.randint(0, len(filerevs) - 1)]
|
|
filerevs.append(node)
|
|
blobs[(filename, node, lastnode)] = content
|
|
revisions.append((filename, node, lastnode, content))
|
|
|
|
pack = self.createPack(revisions)
|
|
|
|
# Verify the pack contents
|
|
for (filename, node, lastnode), content in sorted(blobs.iteritems()):
|
|
chain = pack.getdeltachain(filename, node)
|
|
for entry in chain:
|
|
expectedcontent = blobs[(entry[0], entry[1], entry[3])]
|
|
self.assertEquals(entry[4], expectedcontent)
|
|
|
|
def testPackMetadata(self):
|
|
revisions = []
|
|
for i in range(100):
|
|
filename = "%s.txt" % i
|
|
content = "put-something-here \n" * i
|
|
node = self.getHash(content)
|
|
meta = {constants.METAKEYFLAG: i ** 4, constants.METAKEYSIZE: len(content)}
|
|
revisions.append((filename, node, nullid, content, meta))
|
|
pack = self.createPack(revisions, version=1)
|
|
for name, node, x, content, origmeta in revisions:
|
|
parsedmeta = pack.getmeta(name, node)
|
|
# flag == 0 should be optimized out
|
|
if origmeta[constants.METAKEYFLAG] == 0:
|
|
del origmeta[constants.METAKEYFLAG]
|
|
self.assertEquals(parsedmeta, origmeta)
|
|
|
|
def testPackMetadataThrows(self):
|
|
filename = "1"
|
|
content = "2"
|
|
node = self.getHash(content)
|
|
meta = {constants.METAKEYFLAG: 3}
|
|
revisions = [(filename, node, nullid, content, meta)]
|
|
try:
|
|
self.createPack(revisions, version=0)
|
|
self.assertTrue(False, "should throw if metadata is not supported")
|
|
except RuntimeError:
|
|
pass
|
|
|
|
def testGetMissing(self):
|
|
"""Test the getmissing() api.
|
|
"""
|
|
revisions = []
|
|
filename = "foo"
|
|
lastnode = nullid
|
|
for i in range(10):
|
|
content = "abcdef%s" % i
|
|
node = self.getHash(content)
|
|
revisions.append((filename, node, lastnode, content))
|
|
lastnode = node
|
|
|
|
pack = self.createPack(revisions)
|
|
|
|
missing = pack.getmissing([("foo", revisions[0][1])])
|
|
self.assertFalse(missing)
|
|
|
|
missing = pack.getmissing([("foo", revisions[0][1]), ("foo", revisions[1][1])])
|
|
self.assertFalse(missing)
|
|
|
|
fakenode = self.getFakeHash()
|
|
missing = pack.getmissing([("foo", revisions[0][1]), ("foo", fakenode)])
|
|
self.assertEquals(missing, [("foo", fakenode)])
|
|
|
|
def testAddThrows(self):
|
|
pack = self.createPack()
|
|
|
|
try:
|
|
pack.add("filename", nullid, "contents")
|
|
self.assertTrue(False, "datapack.add should throw")
|
|
except (AttributeError, RuntimeError):
|
|
pass
|
|
|
|
def testBadVersionThrows(self):
|
|
pack = self.createPack()
|
|
path = pack.path() + ".datapack"
|
|
with open(path) as f:
|
|
raw = f.read()
|
|
raw = struct.pack("!B", 255) + raw[1:]
|
|
os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
|
|
with open(path, "w+") as f:
|
|
f.write(raw)
|
|
|
|
try:
|
|
pack = self.datapackreader(pack.path())
|
|
self.assertTrue(False, "bad version number should have thrown")
|
|
except (RuntimeError, MemoryError):
|
|
pass
|
|
|
|
def testMissingDeltabase(self):
|
|
fakenode = self.getFakeHash()
|
|
revisions = [("filename", fakenode, self.getFakeHash(), "content")]
|
|
pack = self.createPack(revisions)
|
|
chain = pack.getdeltachain("filename", fakenode)
|
|
self.assertEquals(len(chain), 1)
|
|
|
|
def testLargePack(self):
|
|
"""Test creating and reading from a large pack with over X entries.
|
|
This causes it to use a 2^16 fanout table instead."""
|
|
revisions = []
|
|
blobs = {}
|
|
total = SMALLFANOUTCUTOFF + 1
|
|
for i in xrange(total):
|
|
filename = "filename-%s" % i
|
|
content = filename
|
|
node = self.getHash(content)
|
|
blobs[(filename, node)] = content
|
|
revisions.append((filename, node, nullid, content))
|
|
|
|
pack = self.createPack(revisions)
|
|
if self.paramsavailable:
|
|
self.assertEquals(pack.params.fanoutprefix, LARGEFANOUTPREFIX)
|
|
|
|
for (filename, node), content in blobs.iteritems():
|
|
actualcontent = pack.getdeltachain(filename, node)[0][4]
|
|
self.assertEquals(actualcontent, content)
|
|
|
|
def testPacksCache(self):
|
|
"""Test that we remember the most recent packs while fetching the delta
|
|
chain."""
|
|
|
|
packdir = self.makeTempDir()
|
|
deltachains = []
|
|
|
|
if self.iscdatapack:
|
|
numpacks = 200
|
|
else:
|
|
numpacks = 10
|
|
revisionsperpack = 100
|
|
|
|
for i in range(numpacks):
|
|
chain = []
|
|
revision = (str(i), self.getFakeHash(), nullid, "content")
|
|
|
|
for _ in range(revisionsperpack):
|
|
chain.append(revision)
|
|
revision = (str(i), self.getFakeHash(), revision[1], self.getFakeHash())
|
|
|
|
self.createPack(chain, packdir)
|
|
deltachains.append(chain)
|
|
|
|
class testdatapackstore(datapackstore):
|
|
# Ensures that we are not keeping everything in the cache.
|
|
DEFAULTCACHESIZE = numpacks / 2
|
|
|
|
store = testdatapackstore(uimod.ui(), packdir, self.iscdatapack)
|
|
|
|
random.shuffle(deltachains)
|
|
for randomchain in deltachains:
|
|
revision = random.choice(randomchain)
|
|
chain = store.getdeltachain(revision[0], revision[1])
|
|
|
|
mostrecentpack = next(iter(store.packs), None)
|
|
self.assertEquals(
|
|
mostrecentpack.getdeltachain(revision[0], revision[1]), chain
|
|
)
|
|
|
|
self.assertEquals(randomchain.index(revision) + 1, len(chain))
|
|
|
|
def testInlineRepack(self):
|
|
"""Verify that when fetchpacks is enabled, and the number of packfiles
|
|
is over DEFAULTCACHESIZE, the refresh operation will trigger a repack,
|
|
reducing the number of packfiles in the store.
|
|
"""
|
|
packdir = self.makeTempDir()
|
|
|
|
numpacks = 20
|
|
revisionsperpack = 100
|
|
|
|
for i in range(numpacks):
|
|
chain = []
|
|
revision = (str(i), self.getFakeHash(), nullid, "content")
|
|
|
|
for _ in range(revisionsperpack):
|
|
chain.append(revision)
|
|
revision = (str(i), self.getFakeHash(), revision[1], self.getFakeHash())
|
|
|
|
self.createPack(chain, packdir)
|
|
|
|
packreader = self.datapackreader
|
|
|
|
class testdatapackstore(datapackstore):
|
|
DEFAULTCACHESIZE = numpacks / 2
|
|
|
|
def getpack(self, path):
|
|
return packreader(path)
|
|
|
|
store = testdatapackstore(uimod.ui(), packdir, self.iscdatapack)
|
|
|
|
# The first refresh should populate all the packfiles.
|
|
store.refresh()
|
|
self.assertEquals(len(store.packs), testdatapackstore.DEFAULTCACHESIZE)
|
|
|
|
# Each packfile is made up of 2 files: the data, and the index
|
|
self.assertEquals(len(os.listdir(packdir)), numpacks * 2)
|
|
|
|
store.markforrefresh()
|
|
|
|
# The second one should repack all the packfiles into one.
|
|
store.fetchpacksenabled = True
|
|
store.refresh()
|
|
self.assertEquals(len(store.packs), 1)
|
|
|
|
# There should only be 2 files: the packfile, and the index
|
|
self.assertEquals(len(os.listdir(packdir)), 2)
|
|
|
|
def testCorruptPackHandling(self):
|
|
"""Test that the pack store deletes corrupt packs."""
|
|
# There's a bug in cdatapack right now that causes it to return bad data
|
|
# even if the pack is corrupt. Since we're not getting an exception, we
|
|
# can't detect the corruption and remediate. Let's wait for the rust
|
|
# implementation to deprecate the C implementation then this will be
|
|
# easier to fix.
|
|
if self.iscdatapack:
|
|
return
|
|
|
|
packdir = self.makeTempDir()
|
|
deltachains = []
|
|
|
|
numpacks = 5
|
|
revisionsperpack = 100
|
|
|
|
firstpack = None
|
|
secondindex = None
|
|
for i in range(numpacks):
|
|
chain = []
|
|
revision = (str(i), self.getFakeHash(), nullid, "content")
|
|
|
|
for _ in range(revisionsperpack):
|
|
chain.append(revision)
|
|
revision = (str(i), self.getFakeHash(), revision[1], self.getFakeHash())
|
|
|
|
pack = self.createPack(chain, packdir)
|
|
if firstpack is None:
|
|
firstpack = pack.packpath()
|
|
elif secondindex is None:
|
|
secondindex = pack.indexpath()
|
|
|
|
deltachains.append(chain)
|
|
|
|
ui = uimod.ui()
|
|
store = datapackstore(ui, packdir, self.iscdatapack, deletecorruptpacks=True)
|
|
|
|
key = (deltachains[0][0][0], deltachains[0][0][1])
|
|
# Count packs
|
|
origpackcount = len(os.listdir(packdir))
|
|
|
|
# Read key
|
|
store.getdelta(*key)
|
|
|
|
# Corrupt the pack
|
|
os.chmod(firstpack, 0o644)
|
|
f = open(firstpack, "w")
|
|
f.truncate(1)
|
|
f.close()
|
|
|
|
# Re-create the store. Otherwise the behavior is kind of "undefined"
|
|
# because the size of mmap-ed memory isn't truncated automatically,
|
|
# and is filled by 0.
|
|
store = datapackstore(ui, packdir, self.iscdatapack, deletecorruptpacks=True)
|
|
|
|
# Look for key again
|
|
try:
|
|
ui.pushbuffer(error=True)
|
|
delta = store.getdelta(*key)
|
|
raise RuntimeError("getdelta on corrupt key should fail %s" % repr(delta))
|
|
except KeyError:
|
|
pass
|
|
ui.popbuffer()
|
|
|
|
# Count packs
|
|
newpackcount = len(os.listdir(packdir))
|
|
|
|
# Assert the corrupt pack was removed
|
|
self.assertEquals(origpackcount - 2, newpackcount)
|
|
|
|
# Corrupt the index
|
|
os.chmod(secondindex, 0o644)
|
|
f = open(secondindex, "w")
|
|
f.truncate(1)
|
|
f.close()
|
|
|
|
# Load the packs
|
|
origpackcount = len(os.listdir(packdir))
|
|
ui.pushbuffer(error=True)
|
|
store = datapackstore(ui, packdir, self.iscdatapack, deletecorruptpacks=True)
|
|
# Constructing the store doesn't load the packfiles, these are loaded
|
|
# on demand, and thus the detection of bad packfiles only happen then.
|
|
# Let's force a refresh to make sure the bad pack files are deleted.
|
|
store.refresh()
|
|
ui.popbuffer()
|
|
newpackcount = len(os.listdir(packdir))
|
|
|
|
# Assert the corrupt pack was removed
|
|
self.assertEquals(origpackcount - 2, newpackcount)
|
|
|
|
def testReadingMutablePack(self):
|
|
"""Tests that the data written into a mutabledatapack can be read out
|
|
before it has been finalized."""
|
|
packdir = self.makeTempDir()
|
|
packer = mutabledatapack(uimod.ui(), packdir, version=1)
|
|
|
|
# Add some unused first revision for noise
|
|
packer.add("qwert", self.getFakeHash(), self.getFakeHash(), "qwertcontent")
|
|
|
|
filename = "filename1"
|
|
node = self.getFakeHash()
|
|
base = self.getFakeHash()
|
|
content = "asdf"
|
|
meta = {
|
|
constants.METAKEYFLAG: 1,
|
|
constants.METAKEYSIZE: len(content),
|
|
"Z": "random_string",
|
|
"_": "\0" * 40,
|
|
}
|
|
packer.add(filename, node, base, content, metadata=meta)
|
|
|
|
# Add some unused third revision for noise
|
|
packer.add("zxcv", self.getFakeHash(), self.getFakeHash(), "zcxvcontent")
|
|
|
|
# Test getmissing
|
|
missing = ("", self.getFakeHash())
|
|
value = packer.getmissing([missing, (filename, node)])
|
|
self.assertEquals(value, [missing])
|
|
|
|
# Test getmeta
|
|
value = packer.getmeta(filename, node)
|
|
self.assertEquals(value, meta)
|
|
|
|
# Test getdelta
|
|
value = packer.getdelta(filename, node)
|
|
self.assertEquals(value, (content, filename, base, meta))
|
|
|
|
# Test getdeltachain
|
|
value = packer.getdeltachain(filename, node)
|
|
self.assertEquals(value, [(filename, node, filename, base, content)])
|
|
|
|
# perf test off by default since it's slow
|
|
def _testIndexPerf(self):
|
|
random.seed(0)
|
|
print("Multi-get perf test")
|
|
packsizes = [100, 10000, 100000, 500000, 1000000, 3000000]
|
|
lookupsizes = [10, 100, 1000, 10000, 100000, 1000000]
|
|
for packsize in packsizes:
|
|
revisions = []
|
|
for i in xrange(packsize):
|
|
filename = "filename-%s" % i
|
|
content = "content-%s" % i
|
|
node = self.getHash(content)
|
|
revisions.append((filename, node, nullid, content))
|
|
|
|
path = self.createPack(revisions).path()
|
|
|
|
# Perf of large multi-get
|
|
import gc
|
|
|
|
gc.disable()
|
|
pack = self.datapackreader(path)
|
|
for lookupsize in lookupsizes:
|
|
if lookupsize > packsize:
|
|
continue
|
|
random.shuffle(revisions)
|
|
findnodes = [(rev[0], rev[1]) for rev in revisions]
|
|
|
|
start = time.time()
|
|
pack.getmissing(findnodes[:lookupsize])
|
|
elapsed = time.time() - start
|
|
print(
|
|
"%s pack %s lookups = %0.04f"
|
|
% (
|
|
("%s" % packsize).rjust(7),
|
|
("%s" % lookupsize).rjust(7),
|
|
elapsed,
|
|
)
|
|
)
|
|
|
|
print("")
|
|
gc.enable()
|
|
|
|
# The perf test is meant to produce output, so we always fail the test
|
|
# so the user sees the output.
|
|
raise RuntimeError("perf test always fails")
|
|
|
|
|
|
class datapacktests(datapacktestsbase, unittest.TestCase):
|
|
def __init__(self, *args, **kwargs):
|
|
datapacktestsbase.__init__(self, datapack, True, False)
|
|
unittest.TestCase.__init__(self, *args, **kwargs)
|
|
|
|
|
|
class fastdatapacktests(datapacktestsbase, unittest.TestCase):
|
|
def __init__(self, *args, **kwargs):
|
|
datapacktestsbase.__init__(self, fastdatapack, False, True)
|
|
unittest.TestCase.__init__(self, *args, **kwargs)
|
|
|
|
|
|
class rustdatapacktests(datapacktestsbase, unittest.TestCase):
|
|
def __init__(self, *args, **kwargs):
|
|
datapacktestsbase.__init__(self, rustdatapack, False, True)
|
|
unittest.TestCase.__init__(self, *args, **kwargs)
|
|
|
|
|
|
# TODO:
|
|
# datapack store:
|
|
# - getmissing
|
|
# - GC two packs into one
|
|
|
|
if __name__ == "__main__":
|
|
silenttestrunner.main(__name__)
|