2017-02-24 01:03:02 +03:00
|
|
|
#!/usr/bin/env python
|
2017-11-16 21:56:44 +03:00
|
|
|
from __future__ import absolute_import, print_function
|
|
|
|
|
2016-06-16 01:48:16 +03:00
|
|
|
import hashlib
|
2017-01-13 20:42:25 +03:00
|
|
|
import os
|
2016-04-29 01:00:34 +03:00
|
|
|
import random
|
|
|
|
import shutil
|
2017-01-13 20:42:25 +03:00
|
|
|
import stat
|
2016-04-29 01:00:34 +03:00
|
|
|
import struct
|
|
|
|
import tempfile
|
2016-05-03 22:32:16 +03:00
|
|
|
import time
|
2016-04-29 01:00:34 +03:00
|
|
|
import unittest
|
|
|
|
|
2019-01-29 05:29:48 +03:00
|
|
|
import edenscm.mercurial.ui as uimod
|
2016-04-29 01:00:34 +03:00
|
|
|
import silenttestrunner
|
2019-01-30 03:25:33 +03:00
|
|
|
from edenscm.hgext.extlib.pyrevisionstore import datapack as rustdatapack
|
|
|
|
from edenscm.hgext.remotefilelog import constants
|
|
|
|
from edenscm.hgext.remotefilelog.basepack import (
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
LARGEFANOUTPREFIX,
|
|
|
|
SMALLFANOUTCUTOFF,
|
|
|
|
SMALLFANOUTPREFIX,
|
|
|
|
)
|
2019-01-30 03:25:33 +03:00
|
|
|
from edenscm.hgext.remotefilelog.datapack import (
|
2016-05-16 20:59:09 +03:00
|
|
|
datapack,
|
2017-11-08 21:25:12 +03:00
|
|
|
datapackstore,
|
2016-08-31 02:19:53 +03:00
|
|
|
fastdatapack,
|
2016-05-16 20:59:09 +03:00
|
|
|
mutabledatapack,
|
2016-05-20 19:31:37 +03:00
|
|
|
)
|
2019-01-30 03:25:33 +03:00
|
|
|
from edenscm.mercurial.node import nullid
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2016-04-29 01:00:34 +03:00
|
|
|
|
flake8: enable F821 check
Summary:
This check is useful and detects real errors (ex. fbconduit). Unfortunately
`arc lint` will run it with both py2 and py3 so a lot of py2 builtins will
still be warned.
I didn't find a clean way to disable py3 check. So this diff tries to fix them.
For `xrange`, the change was done by a script:
```
import sys
import redbaron
headertypes = {'comment', 'endl', 'from_import', 'import', 'string',
'assignment', 'atomtrailers'}
xrangefix = '''try:
xrange(0)
except NameError:
xrange = range
'''
def isxrange(x):
try:
return x[0].value == 'xrange'
except Exception:
return False
def main(argv):
for i, path in enumerate(argv):
print('(%d/%d) scanning %s' % (i + 1, len(argv), path))
content = open(path).read()
try:
red = redbaron.RedBaron(content)
except Exception:
print(' warning: failed to parse')
continue
hasxrange = red.find('atomtrailersnode', value=isxrange)
hasxrangefix = 'xrange = range' in content
if hasxrangefix or not hasxrange:
print(' no need to change')
continue
# find a place to insert the compatibility statement
changed = False
for node in red:
if node.type in headertypes:
continue
# node.insert_before is an easier API, but it has bugs changing
# other "finally" and "except" positions. So do the insert
# manually.
# # node.insert_before(xrangefix)
line = node.absolute_bounding_box.top_left.line - 1
lines = content.splitlines(1)
content = ''.join(lines[:line]) + xrangefix + ''.join(lines[line:])
changed = True
break
if changed:
# "content" is faster than "red.dumps()"
open(path, 'w').write(content)
print(' updated')
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
```
For other py2 builtins that do not have a py3 equivalent, some `# noqa`
were added as a workaround for now.
Reviewed By: DurhamG
Differential Revision: D6934535
fbshipit-source-id: 546b62830af144bc8b46788d2e0fd00496838939
2018-02-10 04:31:44 +03:00
|
|
|
try:
|
|
|
|
xrange(0)
|
|
|
|
except NameError:
|
|
|
|
xrange = range
|
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2016-08-31 02:19:53 +03:00
|
|
|
class datapacktestsbase(object):
|
2017-11-17 08:24:44 +03:00
|
|
|
def __init__(self, datapackreader, paramsavailable, iscdatapack):
|
2016-08-31 02:19:53 +03:00
|
|
|
self.datapackreader = datapackreader
|
2017-11-17 08:24:44 +03:00
|
|
|
self.iscdatapack = iscdatapack
|
2016-08-31 02:19:53 +03:00
|
|
|
self.paramsavailable = paramsavailable
|
|
|
|
|
2016-04-29 01:00:34 +03:00
|
|
|
def setUp(self):
|
|
|
|
self.tempdirs = []
|
|
|
|
|
|
|
|
def tearDown(self):
|
|
|
|
for d in self.tempdirs:
|
|
|
|
shutil.rmtree(d)
|
|
|
|
|
|
|
|
def makeTempDir(self):
|
|
|
|
tempdir = tempfile.mkdtemp()
|
|
|
|
self.tempdirs.append(tempdir)
|
|
|
|
return tempdir
|
|
|
|
|
|
|
|
def getHash(self, content):
|
2016-06-16 01:48:16 +03:00
|
|
|
return hashlib.sha1(content).digest()
|
2016-04-29 01:00:34 +03:00
|
|
|
|
|
|
|
def getFakeHash(self):
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
return "".join(chr(random.randint(0, 255)) for _ in range(20))
|
2016-04-29 01:00:34 +03:00
|
|
|
|
2017-11-08 21:25:12 +03:00
|
|
|
def createPack(self, revisions=None, packdir=None, version=0):
|
2016-04-29 01:00:34 +03:00
|
|
|
if revisions is None:
|
|
|
|
revisions = [("filename", self.getFakeHash(), nullid, "content")]
|
|
|
|
|
2017-11-08 21:25:12 +03:00
|
|
|
if packdir is None:
|
|
|
|
packdir = self.makeTempDir()
|
|
|
|
|
2019-01-29 05:29:48 +03:00
|
|
|
packer = mutabledatapack(uimod.ui(), packdir, version=version)
|
2016-04-29 01:00:34 +03:00
|
|
|
|
2017-04-27 05:50:36 +03:00
|
|
|
for args in revisions:
|
|
|
|
filename, node, base, content = args[0:4]
|
|
|
|
# meta is optional
|
|
|
|
meta = None
|
|
|
|
if len(args) > 4:
|
|
|
|
meta = args[4]
|
|
|
|
packer.add(filename, node, base, content, metadata=meta)
|
2016-04-29 01:00:34 +03:00
|
|
|
|
|
|
|
path = packer.close()
|
2016-08-31 02:19:53 +03:00
|
|
|
return self.datapackreader(path)
|
2016-04-29 01:00:34 +03:00
|
|
|
|
2017-05-02 20:01:13 +03:00
|
|
|
def _testAddSingle(self, content):
|
2016-04-29 01:00:34 +03:00
|
|
|
"""Test putting a simple blob into a pack and reading it out.
|
|
|
|
"""
|
|
|
|
filename = "foo"
|
|
|
|
node = self.getHash(content)
|
|
|
|
|
|
|
|
revisions = [(filename, node, nullid, content)]
|
|
|
|
pack = self.createPack(revisions)
|
2016-08-31 02:19:53 +03:00
|
|
|
if self.paramsavailable:
|
|
|
|
self.assertEquals(pack.params.fanoutprefix, SMALLFANOUTPREFIX)
|
2016-04-29 01:00:34 +03:00
|
|
|
|
|
|
|
chain = pack.getdeltachain(filename, node)
|
|
|
|
self.assertEquals(content, chain[0][4])
|
|
|
|
|
2017-05-02 20:01:13 +03:00
|
|
|
def testAddSingle(self):
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
self._testAddSingle("")
|
2017-05-02 20:01:13 +03:00
|
|
|
|
|
|
|
def testAddSingleEmpty(self):
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
self._testAddSingle("abcdef")
|
2017-05-02 20:01:13 +03:00
|
|
|
|
2016-04-29 01:00:34 +03:00
|
|
|
def testAddMultiple(self):
|
|
|
|
"""Test putting multiple unrelated blobs into a pack and reading them
|
|
|
|
out.
|
|
|
|
"""
|
|
|
|
revisions = []
|
|
|
|
for i in range(10):
|
|
|
|
filename = "foo%s" % i
|
|
|
|
content = "abcdef%s" % i
|
|
|
|
node = self.getHash(content)
|
2017-12-15 22:10:31 +03:00
|
|
|
revisions.append((filename, node, self.getFakeHash(), content))
|
2016-04-29 01:00:34 +03:00
|
|
|
|
|
|
|
pack = self.createPack(revisions)
|
|
|
|
|
|
|
|
for filename, node, base, content in revisions:
|
2017-12-15 22:10:31 +03:00
|
|
|
entry = pack.getdelta(filename, node)
|
|
|
|
self.assertEquals((content, filename, base, {}), entry)
|
|
|
|
|
2016-04-29 01:00:34 +03:00
|
|
|
chain = pack.getdeltachain(filename, node)
|
|
|
|
self.assertEquals(content, chain[0][4])
|
|
|
|
|
|
|
|
def testAddDeltas(self):
|
|
|
|
"""Test putting multiple delta blobs into a pack and read the chain.
|
|
|
|
"""
|
|
|
|
revisions = []
|
|
|
|
filename = "foo"
|
|
|
|
lastnode = nullid
|
|
|
|
for i in range(10):
|
|
|
|
content = "abcdef%s" % i
|
|
|
|
node = self.getHash(content)
|
|
|
|
revisions.append((filename, node, lastnode, content))
|
|
|
|
lastnode = node
|
|
|
|
|
|
|
|
pack = self.createPack(revisions)
|
2017-12-15 22:10:31 +03:00
|
|
|
|
|
|
|
entry = pack.getdelta(filename, revisions[0][1])
|
|
|
|
realvalue = (revisions[0][3], filename, revisions[0][2], {})
|
|
|
|
self.assertEquals(entry, realvalue)
|
|
|
|
|
2016-04-29 01:00:34 +03:00
|
|
|
# Test that the chain for the final entry has all the others
|
|
|
|
chain = pack.getdeltachain(filename, node)
|
|
|
|
for i in range(10):
|
|
|
|
content = "abcdef%s" % i
|
|
|
|
self.assertEquals(content, chain[-i - 1][4])
|
|
|
|
|
|
|
|
def testPackMany(self):
|
|
|
|
"""Pack many related and unrelated objects.
|
|
|
|
"""
|
|
|
|
# Build a random pack file
|
|
|
|
revisions = []
|
|
|
|
blobs = {}
|
|
|
|
random.seed(0)
|
|
|
|
for i in range(100):
|
|
|
|
filename = "filename-%s" % i
|
|
|
|
filerevs = []
|
|
|
|
for j in range(random.randint(1, 100)):
|
2018-07-25 20:56:54 +03:00
|
|
|
content = "content-%s-%s" % (i, j)
|
2016-04-29 01:00:34 +03:00
|
|
|
node = self.getHash(content)
|
|
|
|
lastnode = nullid
|
|
|
|
if len(filerevs) > 0:
|
|
|
|
lastnode = filerevs[random.randint(0, len(filerevs) - 1)]
|
|
|
|
filerevs.append(node)
|
|
|
|
blobs[(filename, node, lastnode)] = content
|
|
|
|
revisions.append((filename, node, lastnode, content))
|
|
|
|
|
|
|
|
pack = self.createPack(revisions)
|
|
|
|
|
|
|
|
# Verify the pack contents
|
|
|
|
for (filename, node, lastnode), content in sorted(blobs.iteritems()):
|
|
|
|
chain = pack.getdeltachain(filename, node)
|
|
|
|
for entry in chain:
|
|
|
|
expectedcontent = blobs[(entry[0], entry[1], entry[3])]
|
|
|
|
self.assertEquals(entry[4], expectedcontent)
|
|
|
|
|
2017-04-27 05:50:36 +03:00
|
|
|
def testPackMetadata(self):
|
|
|
|
revisions = []
|
2017-05-02 05:03:25 +03:00
|
|
|
for i in range(100):
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
filename = "%s.txt" % i
|
|
|
|
content = "put-something-here \n" * i
|
2017-04-27 05:50:36 +03:00
|
|
|
node = self.getHash(content)
|
2018-07-25 20:56:54 +03:00
|
|
|
meta = {constants.METAKEYFLAG: i ** 4, constants.METAKEYSIZE: len(content)}
|
2017-04-27 05:50:36 +03:00
|
|
|
revisions.append((filename, node, nullid, content, meta))
|
|
|
|
pack = self.createPack(revisions, version=1)
|
|
|
|
for name, node, x, content, origmeta in revisions:
|
|
|
|
parsedmeta = pack.getmeta(name, node)
|
|
|
|
# flag == 0 should be optimized out
|
|
|
|
if origmeta[constants.METAKEYFLAG] == 0:
|
|
|
|
del origmeta[constants.METAKEYFLAG]
|
|
|
|
self.assertEquals(parsedmeta, origmeta)
|
|
|
|
|
|
|
|
def testPackMetadataThrows(self):
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
filename = "1"
|
|
|
|
content = "2"
|
2017-04-27 05:50:36 +03:00
|
|
|
node = self.getHash(content)
|
|
|
|
meta = {constants.METAKEYFLAG: 3}
|
|
|
|
revisions = [(filename, node, nullid, content, meta)]
|
|
|
|
try:
|
|
|
|
self.createPack(revisions, version=0)
|
|
|
|
self.assertTrue(False, "should throw if metadata is not supported")
|
|
|
|
except RuntimeError:
|
|
|
|
pass
|
|
|
|
|
2016-04-29 01:00:34 +03:00
|
|
|
def testGetMissing(self):
|
|
|
|
"""Test the getmissing() api.
|
|
|
|
"""
|
|
|
|
revisions = []
|
|
|
|
filename = "foo"
|
|
|
|
lastnode = nullid
|
|
|
|
for i in range(10):
|
|
|
|
content = "abcdef%s" % i
|
|
|
|
node = self.getHash(content)
|
|
|
|
revisions.append((filename, node, lastnode, content))
|
|
|
|
lastnode = node
|
|
|
|
|
|
|
|
pack = self.createPack(revisions)
|
|
|
|
|
|
|
|
missing = pack.getmissing([("foo", revisions[0][1])])
|
|
|
|
self.assertFalse(missing)
|
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
missing = pack.getmissing([("foo", revisions[0][1]), ("foo", revisions[1][1])])
|
2016-04-29 01:00:34 +03:00
|
|
|
self.assertFalse(missing)
|
|
|
|
|
|
|
|
fakenode = self.getFakeHash()
|
|
|
|
missing = pack.getmissing([("foo", revisions[0][1]), ("foo", fakenode)])
|
|
|
|
self.assertEquals(missing, [("foo", fakenode)])
|
|
|
|
|
|
|
|
def testAddThrows(self):
|
|
|
|
pack = self.createPack()
|
|
|
|
|
|
|
|
try:
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
pack.add("filename", nullid, "contents")
|
2016-04-29 01:00:34 +03:00
|
|
|
self.assertTrue(False, "datapack.add should throw")
|
2018-07-25 20:56:54 +03:00
|
|
|
except (AttributeError, RuntimeError):
|
2016-04-29 01:00:34 +03:00
|
|
|
pass
|
|
|
|
|
|
|
|
def testBadVersionThrows(self):
|
|
|
|
pack = self.createPack()
|
2018-07-10 03:39:36 +03:00
|
|
|
path = pack.path() + ".datapack"
|
2016-04-29 01:00:34 +03:00
|
|
|
with open(path) as f:
|
|
|
|
raw = f.read()
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
raw = struct.pack("!B", 255) + raw[1:]
|
2017-01-13 20:42:25 +03:00
|
|
|
os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
with open(path, "w+") as f:
|
2016-04-29 01:00:34 +03:00
|
|
|
f.write(raw)
|
|
|
|
|
|
|
|
try:
|
2018-07-10 03:39:36 +03:00
|
|
|
pack = self.datapackreader(pack.path())
|
2016-04-29 01:00:34 +03:00
|
|
|
self.assertTrue(False, "bad version number should have thrown")
|
|
|
|
except RuntimeError:
|
|
|
|
pass
|
|
|
|
|
2016-05-16 20:59:09 +03:00
|
|
|
def testMissingDeltabase(self):
|
|
|
|
fakenode = self.getFakeHash()
|
|
|
|
revisions = [("filename", fakenode, self.getFakeHash(), "content")]
|
|
|
|
pack = self.createPack(revisions)
|
|
|
|
chain = pack.getdeltachain("filename", fakenode)
|
|
|
|
self.assertEquals(len(chain), 1)
|
|
|
|
|
2016-05-16 20:59:09 +03:00
|
|
|
def testLargePack(self):
|
|
|
|
"""Test creating and reading from a large pack with over X entries.
|
|
|
|
This causes it to use a 2^16 fanout table instead."""
|
|
|
|
revisions = []
|
|
|
|
blobs = {}
|
|
|
|
total = SMALLFANOUTCUTOFF + 1
|
|
|
|
for i in xrange(total):
|
|
|
|
filename = "filename-%s" % i
|
|
|
|
content = filename
|
|
|
|
node = self.getHash(content)
|
|
|
|
blobs[(filename, node)] = content
|
|
|
|
revisions.append((filename, node, nullid, content))
|
|
|
|
|
|
|
|
pack = self.createPack(revisions)
|
2016-08-31 02:19:53 +03:00
|
|
|
if self.paramsavailable:
|
|
|
|
self.assertEquals(pack.params.fanoutprefix, LARGEFANOUTPREFIX)
|
2016-05-16 20:59:09 +03:00
|
|
|
|
|
|
|
for (filename, node), content in blobs.iteritems():
|
|
|
|
actualcontent = pack.getdeltachain(filename, node)[0][4]
|
|
|
|
self.assertEquals(actualcontent, content)
|
|
|
|
|
2017-11-08 21:25:12 +03:00
|
|
|
def testPacksCache(self):
|
|
|
|
"""Test that we remember the most recent packs while fetching the delta
|
|
|
|
chain."""
|
|
|
|
|
|
|
|
packdir = self.makeTempDir()
|
|
|
|
deltachains = []
|
|
|
|
|
2017-11-17 08:24:44 +03:00
|
|
|
if self.iscdatapack:
|
|
|
|
numpacks = 200
|
|
|
|
else:
|
|
|
|
numpacks = 10
|
2017-11-08 21:25:12 +03:00
|
|
|
revisionsperpack = 100
|
|
|
|
|
|
|
|
for i in range(numpacks):
|
|
|
|
chain = []
|
|
|
|
revision = (str(i), self.getFakeHash(), nullid, "content")
|
|
|
|
|
|
|
|
for _ in range(revisionsperpack):
|
|
|
|
chain.append(revision)
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
revision = (str(i), self.getFakeHash(), revision[1], self.getFakeHash())
|
2017-11-08 21:25:12 +03:00
|
|
|
|
|
|
|
self.createPack(chain, packdir)
|
|
|
|
deltachains.append(chain)
|
|
|
|
|
2017-11-16 21:04:36 +03:00
|
|
|
class testdatapackstore(datapackstore):
|
|
|
|
# Ensures that we are not keeping everything in the cache.
|
|
|
|
DEFAULTCACHESIZE = numpacks / 2
|
2017-11-08 21:25:12 +03:00
|
|
|
|
2019-01-29 05:29:48 +03:00
|
|
|
store = testdatapackstore(uimod.ui(), packdir, self.iscdatapack)
|
2017-11-08 21:25:12 +03:00
|
|
|
|
2017-11-16 21:04:36 +03:00
|
|
|
random.shuffle(deltachains)
|
|
|
|
for randomchain in deltachains:
|
|
|
|
revision = random.choice(randomchain)
|
|
|
|
chain = store.getdeltachain(revision[0], revision[1])
|
|
|
|
|
|
|
|
mostrecentpack = next(iter(store.packs), None)
|
|
|
|
self.assertEquals(
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
mostrecentpack.getdeltachain(revision[0], revision[1]), chain
|
2017-11-16 21:04:36 +03:00
|
|
|
)
|
2017-11-08 21:25:12 +03:00
|
|
|
|
2017-11-16 21:04:36 +03:00
|
|
|
self.assertEquals(randomchain.index(revision) + 1, len(chain))
|
2017-11-08 21:25:12 +03:00
|
|
|
|
2018-05-25 20:51:54 +03:00
|
|
|
def testCorruptPackHandling(self):
|
|
|
|
"""Test that the pack store deletes corrupt packs."""
|
|
|
|
# There's a bug in cdatapack right now that causes it to return bad data
|
|
|
|
# even if the pack is corrupt. Since we're not getting an exception, we
|
|
|
|
# can't detect the corruption and remediate. Let's wait for the rust
|
|
|
|
# implementation to deprecate the C implementation then this will be
|
|
|
|
# easier to fix.
|
|
|
|
if self.iscdatapack:
|
|
|
|
return
|
|
|
|
|
|
|
|
packdir = self.makeTempDir()
|
|
|
|
deltachains = []
|
|
|
|
|
|
|
|
numpacks = 5
|
|
|
|
revisionsperpack = 100
|
|
|
|
|
|
|
|
firstpack = None
|
2018-06-08 03:49:13 +03:00
|
|
|
secondindex = None
|
2018-05-25 20:51:54 +03:00
|
|
|
for i in range(numpacks):
|
|
|
|
chain = []
|
|
|
|
revision = (str(i), self.getFakeHash(), nullid, "content")
|
|
|
|
|
|
|
|
for _ in range(revisionsperpack):
|
|
|
|
chain.append(revision)
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
revision = (str(i), self.getFakeHash(), revision[1], self.getFakeHash())
|
2018-05-25 20:51:54 +03:00
|
|
|
|
|
|
|
pack = self.createPack(chain, packdir)
|
|
|
|
if firstpack is None:
|
2018-07-25 20:57:04 +03:00
|
|
|
firstpack = pack.packpath()
|
2018-06-08 03:49:13 +03:00
|
|
|
elif secondindex is None:
|
2018-07-25 20:57:04 +03:00
|
|
|
secondindex = pack.indexpath()
|
2018-06-08 03:49:13 +03:00
|
|
|
|
2018-05-25 20:51:54 +03:00
|
|
|
deltachains.append(chain)
|
|
|
|
|
2019-01-29 05:29:48 +03:00
|
|
|
ui = uimod.ui()
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
store = datapackstore(ui, packdir, self.iscdatapack, deletecorruptpacks=True)
|
2018-05-25 20:51:54 +03:00
|
|
|
|
|
|
|
key = (deltachains[0][0][0], deltachains[0][0][1])
|
|
|
|
# Count packs
|
|
|
|
origpackcount = len(os.listdir(packdir))
|
|
|
|
|
|
|
|
# Read key
|
|
|
|
store.getdelta(*key)
|
|
|
|
|
2018-06-08 03:49:13 +03:00
|
|
|
# Corrupt the pack
|
2018-05-25 20:51:54 +03:00
|
|
|
os.chmod(firstpack, 0o644)
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
f = open(firstpack, "w")
|
2018-05-25 20:51:54 +03:00
|
|
|
f.truncate(1)
|
|
|
|
f.close()
|
|
|
|
|
2018-12-21 04:48:27 +03:00
|
|
|
# Re-create the store. Otherwise the behavior is kind of "undefined"
|
|
|
|
# because the size of mmap-ed memory isn't truncated automatically,
|
|
|
|
# and is filled by 0.
|
|
|
|
store = datapackstore(ui, packdir, self.iscdatapack, deletecorruptpacks=True)
|
|
|
|
|
2018-05-25 20:51:54 +03:00
|
|
|
# Look for key again
|
|
|
|
try:
|
|
|
|
ui.pushbuffer(error=True)
|
|
|
|
delta = store.getdelta(*key)
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
raise RuntimeError("getdelta on corrupt key should fail %s" % repr(delta))
|
2018-05-25 20:51:54 +03:00
|
|
|
except KeyError:
|
|
|
|
pass
|
|
|
|
ui.popbuffer()
|
|
|
|
|
|
|
|
# Count packs
|
|
|
|
newpackcount = len(os.listdir(packdir))
|
|
|
|
|
|
|
|
# Assert the corrupt pack was removed
|
|
|
|
self.assertEquals(origpackcount - 2, newpackcount)
|
|
|
|
|
2018-06-08 03:49:13 +03:00
|
|
|
# Corrupt the index
|
|
|
|
os.chmod(secondindex, 0o644)
|
|
|
|
f = open(secondindex, "w")
|
|
|
|
f.truncate(1)
|
|
|
|
f.close()
|
|
|
|
|
|
|
|
# Load the packs
|
|
|
|
origpackcount = len(os.listdir(packdir))
|
|
|
|
ui.pushbuffer(error=True)
|
|
|
|
store = datapackstore(ui, packdir, self.iscdatapack, deletecorruptpacks=True)
|
2019-02-12 21:53:37 +03:00
|
|
|
# Constructing the store doesn't load the packfiles, these are loaded
|
|
|
|
# on demand, and thus the detection of bad packfiles only happen then.
|
|
|
|
# Let's force a refresh to make sure the bad pack files are deleted.
|
|
|
|
store.refresh()
|
2018-06-08 03:49:13 +03:00
|
|
|
ui.popbuffer()
|
|
|
|
newpackcount = len(os.listdir(packdir))
|
|
|
|
|
|
|
|
# Assert the corrupt pack was removed
|
|
|
|
self.assertEquals(origpackcount - 2, newpackcount)
|
|
|
|
|
2018-02-13 21:12:51 +03:00
|
|
|
def testReadingMutablePack(self):
|
|
|
|
"""Tests that the data written into a mutabledatapack can be read out
|
|
|
|
before it has been finalized."""
|
|
|
|
packdir = self.makeTempDir()
|
2019-01-29 05:29:48 +03:00
|
|
|
packer = mutabledatapack(uimod.ui(), packdir, version=1)
|
2018-02-13 21:12:51 +03:00
|
|
|
|
|
|
|
# Add some unused first revision for noise
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
packer.add("qwert", self.getFakeHash(), self.getFakeHash(), "qwertcontent")
|
2018-02-13 21:12:51 +03:00
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
filename = "filename1"
|
2018-02-13 21:12:51 +03:00
|
|
|
node = self.getFakeHash()
|
|
|
|
base = self.getFakeHash()
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
content = "asdf"
|
|
|
|
meta = {
|
|
|
|
constants.METAKEYFLAG: 1,
|
|
|
|
constants.METAKEYSIZE: len(content),
|
|
|
|
"Z": "random_string",
|
|
|
|
"_": "\0" * 40,
|
|
|
|
}
|
2018-02-13 21:12:51 +03:00
|
|
|
packer.add(filename, node, base, content, metadata=meta)
|
|
|
|
|
|
|
|
# Add some unused third revision for noise
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
packer.add("zxcv", self.getFakeHash(), self.getFakeHash(), "zcxvcontent")
|
2018-02-13 21:12:51 +03:00
|
|
|
|
|
|
|
# Test getmissing
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
missing = ("", self.getFakeHash())
|
2018-02-13 21:12:51 +03:00
|
|
|
value = packer.getmissing([missing, (filename, node)])
|
|
|
|
self.assertEquals(value, [missing])
|
|
|
|
|
|
|
|
# Test getmeta
|
|
|
|
value = packer.getmeta(filename, node)
|
|
|
|
self.assertEquals(value, meta)
|
|
|
|
|
|
|
|
# Test getdelta
|
|
|
|
value = packer.getdelta(filename, node)
|
|
|
|
self.assertEquals(value, (content, filename, base, meta))
|
|
|
|
|
|
|
|
# Test getdeltachain
|
|
|
|
value = packer.getdeltachain(filename, node)
|
|
|
|
self.assertEquals(value, [(filename, node, filename, base, content)])
|
|
|
|
|
2016-05-03 22:32:16 +03:00
|
|
|
# perf test off by default since it's slow
|
|
|
|
def _testIndexPerf(self):
|
|
|
|
random.seed(0)
|
2017-11-16 21:56:44 +03:00
|
|
|
print("Multi-get perf test")
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
packsizes = [100, 10000, 100000, 500000, 1000000, 3000000]
|
|
|
|
lookupsizes = [10, 100, 1000, 10000, 100000, 1000000]
|
2016-05-03 22:32:16 +03:00
|
|
|
for packsize in packsizes:
|
|
|
|
revisions = []
|
|
|
|
for i in xrange(packsize):
|
|
|
|
filename = "filename-%s" % i
|
|
|
|
content = "content-%s" % i
|
|
|
|
node = self.getHash(content)
|
|
|
|
revisions.append((filename, node, nullid, content))
|
|
|
|
|
2018-07-10 03:39:36 +03:00
|
|
|
path = self.createPack(revisions).path()
|
2016-05-03 22:32:16 +03:00
|
|
|
|
|
|
|
# Perf of large multi-get
|
|
|
|
import gc
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2016-05-03 22:32:16 +03:00
|
|
|
gc.disable()
|
2016-08-31 02:19:53 +03:00
|
|
|
pack = self.datapackreader(path)
|
2016-05-03 22:32:16 +03:00
|
|
|
for lookupsize in lookupsizes:
|
|
|
|
if lookupsize > packsize:
|
|
|
|
continue
|
|
|
|
random.shuffle(revisions)
|
|
|
|
findnodes = [(rev[0], rev[1]) for rev in revisions]
|
|
|
|
|
|
|
|
start = time.time()
|
2016-11-25 03:23:21 +03:00
|
|
|
pack.getmissing(findnodes[:lookupsize])
|
2016-05-03 22:32:16 +03:00
|
|
|
elapsed = time.time() - start
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
print(
|
|
|
|
"%s pack %s lookups = %0.04f"
|
|
|
|
% (
|
|
|
|
("%s" % packsize).rjust(7),
|
|
|
|
("%s" % lookupsize).rjust(7),
|
|
|
|
elapsed,
|
|
|
|
)
|
|
|
|
)
|
2016-05-03 22:32:16 +03:00
|
|
|
|
2017-11-16 21:56:44 +03:00
|
|
|
print("")
|
2016-05-03 22:32:16 +03:00
|
|
|
gc.enable()
|
|
|
|
|
|
|
|
# The perf test is meant to produce output, so we always fail the test
|
|
|
|
# so the user sees the output.
|
|
|
|
raise RuntimeError("perf test always fails")
|
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2016-08-31 02:19:53 +03:00
|
|
|
class datapacktests(datapacktestsbase, unittest.TestCase):
|
|
|
|
def __init__(self, *args, **kwargs):
|
2017-11-17 08:24:44 +03:00
|
|
|
datapacktestsbase.__init__(self, datapack, True, False)
|
2016-08-31 02:19:53 +03:00
|
|
|
unittest.TestCase.__init__(self, *args, **kwargs)
|
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2016-08-31 02:19:53 +03:00
|
|
|
class fastdatapacktests(datapacktestsbase, unittest.TestCase):
|
|
|
|
def __init__(self, *args, **kwargs):
|
2017-11-17 08:24:44 +03:00
|
|
|
datapacktestsbase.__init__(self, fastdatapack, False, True)
|
2016-08-31 02:19:53 +03:00
|
|
|
unittest.TestCase.__init__(self, *args, **kwargs)
|
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
|
2018-07-25 20:56:54 +03:00
|
|
|
class rustdatapacktests(datapacktestsbase, unittest.TestCase):
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
|
|
datapacktestsbase.__init__(self, rustdatapack, False, True)
|
|
|
|
unittest.TestCase.__init__(self, *args, **kwargs)
|
|
|
|
|
|
|
|
|
2016-04-29 01:00:34 +03:00
|
|
|
# TODO:
|
|
|
|
# datapack store:
|
|
|
|
# - getmissing
|
|
|
|
# - GC two packs into one
|
|
|
|
|
codemod: join the auto-formatter party
Summary:
Turned on the auto formatter. Ran `arc lint --apply-patches --take BLACK **/*.py`.
Then run `arc lint` again so some other autofixers like spellchecker etc. looked
at the code base. Manually accept the changes whenever they make sense, or use
a workaround (ex. changing "dict()" to "dict constructor") where autofix is false
positive. Disabled linters on files that are hard (i18n/polib.py) to fix, or less
interesting to fix (hgsubversion tests), or cannot be fixed without breaking
OSS build (FBPYTHON4).
Conflicted linters (test-check-module-imports.t, part of test-check-code.t,
test-check-pyflakes.t) are removed or disabled.
Duplicated linters (test-check-pyflakes.t, test-check-pylint.t) are removed.
An issue of the auto-formatter is lines are no longer guarnateed to be <= 80
chars. But that seems less important comparing with the benefit auto-formatter
provides.
As we're here, also remove test-check-py3-compat.t, as it is currently broken
if `PYTHON3=/bin/python3` is set.
Reviewed By: wez, phillco, simpkins, pkaush, singhsrb
Differential Revision: D8173629
fbshipit-source-id: 90e248ae0c5e6eaadbe25520a6ee42d32005621b
2018-05-26 07:34:37 +03:00
|
|
|
if __name__ == "__main__":
|
2016-04-29 01:00:34 +03:00
|
|
|
silenttestrunner.main(__name__)
|