mirror of
https://github.com/facebook/sapling.git
synced 2024-10-11 09:17:30 +03:00
remotefilelog: move constants to class to prepare index format change
Summary: To be able to bump version and change formats, the related constants need to be moved to individual classes. So a class (ex. datapack) can be subclassed to handle different formats. Test Plan: `arc unit` Reviewers: #mercurial, durham Reviewed By: durham Subscribers: mjpieters Differential Revision: https://phabricator.intern.facebook.com/D4927284 Signature: t1:4927284:1493152641:e3274dd735d50baf193b7615dd314f4e6cf161f0
This commit is contained in:
parent
0e2c18e2cd
commit
8fcd86af16
@ -8,7 +8,6 @@ import shallowutil
|
|||||||
# The pack version supported by this implementation. This will need to be
|
# The pack version supported by this implementation. This will need to be
|
||||||
# rev'd whenever the byte format changes. Ex: changing the fanout prefix,
|
# rev'd whenever the byte format changes. Ex: changing the fanout prefix,
|
||||||
# changing any of the int sizes, changing the delta algorithm, etc.
|
# changing any of the int sizes, changing the delta algorithm, etc.
|
||||||
VERSION = 0
|
|
||||||
PACKVERSIONSIZE = 1
|
PACKVERSIONSIZE = 1
|
||||||
INDEXVERSIONSIZE = 2
|
INDEXVERSIONSIZE = 2
|
||||||
|
|
||||||
@ -134,6 +133,7 @@ class basepack(object):
|
|||||||
# The maximum amount we should read via mmap before remmaping so the old
|
# The maximum amount we should read via mmap before remmaping so the old
|
||||||
# pages can be released (100MB)
|
# pages can be released (100MB)
|
||||||
MAXPAGEDIN = 100 * 1024**2
|
MAXPAGEDIN = 100 * 1024**2
|
||||||
|
VERSION = 0
|
||||||
|
|
||||||
def __init__(self, path):
|
def __init__(self, path):
|
||||||
self.path = path
|
self.path = path
|
||||||
@ -151,13 +151,13 @@ class basepack(object):
|
|||||||
self.freememory() # initialize the mmap
|
self.freememory() # initialize the mmap
|
||||||
|
|
||||||
version = struct.unpack('!B', self._data[:PACKVERSIONSIZE])[0]
|
version = struct.unpack('!B', self._data[:PACKVERSIONSIZE])[0]
|
||||||
if version != VERSION:
|
if version != self.VERSION:
|
||||||
raise RuntimeError("unsupported pack version '%s'" %
|
raise RuntimeError("unsupported pack version '%s'" %
|
||||||
version)
|
version)
|
||||||
|
|
||||||
version, config = struct.unpack('!BB',
|
version, config = struct.unpack('!BB',
|
||||||
self._index[:INDEXVERSIONSIZE])
|
self._index[:INDEXVERSIONSIZE])
|
||||||
if version != VERSION:
|
if version != self.VERSION:
|
||||||
raise RuntimeError("unsupported pack index version '%s'" %
|
raise RuntimeError("unsupported pack index version '%s'" %
|
||||||
version)
|
version)
|
||||||
|
|
||||||
@ -210,6 +210,8 @@ class basepack(object):
|
|||||||
raise NotImplemented()
|
raise NotImplemented()
|
||||||
|
|
||||||
class mutablebasepack(object):
|
class mutablebasepack(object):
|
||||||
|
VERSION = 0
|
||||||
|
|
||||||
def __init__(self, ui, packdir):
|
def __init__(self, ui, packdir):
|
||||||
opener = vfsmod.vfs(packdir)
|
opener = vfsmod.vfs(packdir)
|
||||||
opener.createmode = 0o444
|
opener.createmode = 0o444
|
||||||
@ -236,7 +238,7 @@ class mutablebasepack(object):
|
|||||||
# Write header
|
# Write header
|
||||||
# TODO: make it extensible (ex: allow specifying compression algorithm,
|
# TODO: make it extensible (ex: allow specifying compression algorithm,
|
||||||
# a flexible key/value header, delta algorithm, fanout size, etc)
|
# a flexible key/value header, delta algorithm, fanout size, etc)
|
||||||
version = struct.pack('!B', VERSION) # unsigned 1 byte int
|
version = struct.pack('!B', self.VERSION) # unsigned 1 byte int
|
||||||
self.writeraw(version)
|
self.writeraw(version)
|
||||||
|
|
||||||
def __enter__(self):
|
def __enter__(self):
|
||||||
@ -334,7 +336,7 @@ class mutablebasepack(object):
|
|||||||
config = 0
|
config = 0
|
||||||
if indexparams.fanoutprefix == LARGEFANOUTPREFIX:
|
if indexparams.fanoutprefix == LARGEFANOUTPREFIX:
|
||||||
config = 0b10000000
|
config = 0b10000000
|
||||||
self.idxfp.write(struct.pack('!BB', VERSION, config))
|
self.idxfp.write(struct.pack('!BB', self.VERSION, config))
|
||||||
|
|
||||||
class indexparams(object):
|
class indexparams(object):
|
||||||
__slots__ = ('fanoutprefix', 'fanoutstruct', 'fanoutcount', 'fanoutsize',
|
__slots__ = ('fanoutprefix', 'fanoutstruct', 'fanoutcount', 'fanoutsize',
|
||||||
|
@ -10,10 +10,6 @@ try:
|
|||||||
except ImportError:
|
except ImportError:
|
||||||
cstore = None
|
cstore = None
|
||||||
|
|
||||||
# Index entry format is: <node><delta offset><pack data offset><pack data size>
|
|
||||||
# See the mutabledatapack doccomment for more details.
|
|
||||||
INDEXFORMAT = '!20siQQ'
|
|
||||||
INDEXENTRYLENGTH = 40
|
|
||||||
NODELENGTH = 20
|
NODELENGTH = 20
|
||||||
|
|
||||||
# The indicator value in the index for a fulltext entry.
|
# The indicator value in the index for a fulltext entry.
|
||||||
@ -62,6 +58,11 @@ class datapack(basepack.basepack):
|
|||||||
INDEXSUFFIX = INDEXSUFFIX
|
INDEXSUFFIX = INDEXSUFFIX
|
||||||
PACKSUFFIX = PACKSUFFIX
|
PACKSUFFIX = PACKSUFFIX
|
||||||
|
|
||||||
|
# Format is <node><delta offset><pack data offset><pack data size>
|
||||||
|
# See the mutabledatapack doccomment for more details.
|
||||||
|
INDEXFORMAT = '!20siQQ'
|
||||||
|
INDEXENTRYLENGTH = 40
|
||||||
|
|
||||||
def getmissing(self, keys):
|
def getmissing(self, keys):
|
||||||
missing = []
|
missing = []
|
||||||
for name, node in keys:
|
for name, node in keys:
|
||||||
@ -85,11 +86,12 @@ class datapack(basepack.basepack):
|
|||||||
# Precompute chains
|
# Precompute chains
|
||||||
chain = [value]
|
chain = [value]
|
||||||
deltabaseoffset = value[1]
|
deltabaseoffset = value[1]
|
||||||
|
entrylen = self.INDEXENTRYLENGTH
|
||||||
while (deltabaseoffset != FULLTEXTINDEXMARK
|
while (deltabaseoffset != FULLTEXTINDEXMARK
|
||||||
and deltabaseoffset != NOBASEINDEXMARK):
|
and deltabaseoffset != NOBASEINDEXMARK):
|
||||||
loc = params.indexstart + deltabaseoffset
|
loc = params.indexstart + deltabaseoffset
|
||||||
value = struct.unpack(INDEXFORMAT, self._index[loc:loc +
|
value = struct.unpack(self.INDEXFORMAT,
|
||||||
INDEXENTRYLENGTH])
|
self._index[loc:loc + entrylen])
|
||||||
deltabaseoffset = value[1]
|
deltabaseoffset = value[1]
|
||||||
chain.append(value)
|
chain.append(value)
|
||||||
|
|
||||||
@ -148,17 +150,18 @@ class datapack(basepack.basepack):
|
|||||||
index = self._index
|
index = self._index
|
||||||
startnode = index[start:start + NODELENGTH]
|
startnode = index[start:start + NODELENGTH]
|
||||||
endnode = index[end:end + NODELENGTH]
|
endnode = index[end:end + NODELENGTH]
|
||||||
|
entrylen = self.INDEXENTRYLENGTH
|
||||||
if startnode == node:
|
if startnode == node:
|
||||||
entry = index[start:start + INDEXENTRYLENGTH]
|
entry = index[start:start + entrylen]
|
||||||
elif endnode == node:
|
elif endnode == node:
|
||||||
entry = index[end:end + INDEXENTRYLENGTH]
|
entry = index[end:end + entrylen]
|
||||||
else:
|
else:
|
||||||
while start < end - INDEXENTRYLENGTH:
|
while start < end - entrylen:
|
||||||
mid = start + (end - start) / 2
|
mid = start + (end - start) / 2
|
||||||
mid = mid - ((mid - params.indexstart) % INDEXENTRYLENGTH)
|
mid = mid - ((mid - params.indexstart) % entrylen)
|
||||||
midnode = index[mid:mid + NODELENGTH]
|
midnode = index[mid:mid + NODELENGTH]
|
||||||
if midnode == node:
|
if midnode == node:
|
||||||
entry = index[mid:mid + INDEXENTRYLENGTH]
|
entry = index[mid:mid + entrylen]
|
||||||
break
|
break
|
||||||
if node > midnode:
|
if node > midnode:
|
||||||
start = mid
|
start = mid
|
||||||
@ -169,7 +172,7 @@ class datapack(basepack.basepack):
|
|||||||
else:
|
else:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
return struct.unpack(INDEXFORMAT, entry)
|
return struct.unpack(self.INDEXFORMAT, entry)
|
||||||
|
|
||||||
def markledger(self, ledger):
|
def markledger(self, ledger):
|
||||||
for filename, node in self:
|
for filename, node in self:
|
||||||
@ -349,7 +352,10 @@ class mutabledatapack(basepack.mutablebasepack):
|
|||||||
"""
|
"""
|
||||||
INDEXSUFFIX = INDEXSUFFIX
|
INDEXSUFFIX = INDEXSUFFIX
|
||||||
PACKSUFFIX = PACKSUFFIX
|
PACKSUFFIX = PACKSUFFIX
|
||||||
INDEXENTRYLENGTH = INDEXENTRYLENGTH
|
|
||||||
|
# v0 index format: <node><delta offset><pack data offset><pack data size>
|
||||||
|
INDEXFORMAT = datapack.INDEXFORMAT
|
||||||
|
INDEXENTRYLENGTH = datapack.INDEXENTRYLENGTH
|
||||||
|
|
||||||
def add(self, name, node, deltabasenode, delta):
|
def add(self, name, node, deltabasenode, delta):
|
||||||
if len(name) > 2**16:
|
if len(name) > 2**16:
|
||||||
@ -384,6 +390,7 @@ class mutabledatapack(basepack.mutablebasepack):
|
|||||||
in self.entries.iteritems())
|
in self.entries.iteritems())
|
||||||
|
|
||||||
rawindex = ''
|
rawindex = ''
|
||||||
|
fmt = self.INDEXFORMAT
|
||||||
for node, deltabase, offset, size in entries:
|
for node, deltabase, offset, size in entries:
|
||||||
if deltabase == nullid:
|
if deltabase == nullid:
|
||||||
deltabaselocation = FULLTEXTINDEXMARK
|
deltabaselocation = FULLTEXTINDEXMARK
|
||||||
@ -393,8 +400,7 @@ class mutabledatapack(basepack.mutablebasepack):
|
|||||||
deltabaselocation = nodelocations.get(deltabase,
|
deltabaselocation = nodelocations.get(deltabase,
|
||||||
NOBASEINDEXMARK)
|
NOBASEINDEXMARK)
|
||||||
|
|
||||||
entry = struct.pack(INDEXFORMAT, node, deltabaselocation, offset,
|
entry = struct.pack(fmt, node, deltabaselocation, offset, size)
|
||||||
size)
|
|
||||||
rawindex += entry
|
rawindex += entry
|
||||||
|
|
||||||
return rawindex
|
return rawindex
|
||||||
|
@ -53,6 +53,11 @@ class historypack(basepack.basepack):
|
|||||||
INDEXSUFFIX = INDEXSUFFIX
|
INDEXSUFFIX = INDEXSUFFIX
|
||||||
PACKSUFFIX = PACKSUFFIX
|
PACKSUFFIX = PACKSUFFIX
|
||||||
|
|
||||||
|
INDEXFORMAT = INDEXFORMAT
|
||||||
|
INDEXENTRYLENGTH = INDEXENTRYLENGTH
|
||||||
|
|
||||||
|
VERSION = 0
|
||||||
|
|
||||||
def getmissing(self, keys):
|
def getmissing(self, keys):
|
||||||
missing = []
|
missing = []
|
||||||
for name, node in keys:
|
for name, node in keys:
|
||||||
@ -167,17 +172,19 @@ class historypack(basepack.basepack):
|
|||||||
# Bisect between start and end to find node
|
# Bisect between start and end to find node
|
||||||
startnode = self._index[start:start + NODELENGTH]
|
startnode = self._index[start:start + NODELENGTH]
|
||||||
endnode = self._index[end:end + NODELENGTH]
|
endnode = self._index[end:end + NODELENGTH]
|
||||||
|
entrylen = self.INDEXENTRYLENGTH
|
||||||
|
|
||||||
if startnode == namehash:
|
if startnode == namehash:
|
||||||
entry = self._index[start:start + INDEXENTRYLENGTH]
|
entry = self._index[start:start + entrylen]
|
||||||
elif endnode == namehash:
|
elif endnode == namehash:
|
||||||
entry = self._index[end:end + INDEXENTRYLENGTH]
|
entry = self._index[end:end + entrylen]
|
||||||
else:
|
else:
|
||||||
while start < end - INDEXENTRYLENGTH:
|
while start < end - entrylen:
|
||||||
mid = start + (end - start) / 2
|
mid = start + (end - start) / 2
|
||||||
mid = mid - ((mid - params.indexstart) % INDEXENTRYLENGTH)
|
mid = mid - ((mid - params.indexstart) % entrylen)
|
||||||
midnode = self._index[mid:mid + NODELENGTH]
|
midnode = self._index[mid:mid + NODELENGTH]
|
||||||
if midnode == namehash:
|
if midnode == namehash:
|
||||||
entry = self._index[mid:mid + INDEXENTRYLENGTH]
|
entry = self._index[mid:mid + entrylen]
|
||||||
break
|
break
|
||||||
if namehash > midnode:
|
if namehash > midnode:
|
||||||
start = mid
|
start = mid
|
||||||
@ -188,7 +195,7 @@ class historypack(basepack.basepack):
|
|||||||
else:
|
else:
|
||||||
raise KeyError(name)
|
raise KeyError(name)
|
||||||
|
|
||||||
filenamehash, offset, size = struct.unpack(INDEXFORMAT, entry)
|
filenamehash, offset, size = struct.unpack(self.INDEXFORMAT, entry)
|
||||||
filenamelength = struct.unpack('!H', self._data[offset:offset +
|
filenamelength = struct.unpack('!H', self._data[offset:offset +
|
||||||
constants.FILENAMESIZE])[0]
|
constants.FILENAMESIZE])[0]
|
||||||
offset += constants.FILENAMESIZE
|
offset += constants.FILENAMESIZE
|
||||||
@ -318,8 +325,12 @@ class mutablehistorypack(basepack.mutablebasepack):
|
|||||||
"""
|
"""
|
||||||
INDEXSUFFIX = INDEXSUFFIX
|
INDEXSUFFIX = INDEXSUFFIX
|
||||||
PACKSUFFIX = PACKSUFFIX
|
PACKSUFFIX = PACKSUFFIX
|
||||||
|
|
||||||
|
INDEXFORMAT = INDEXFORMAT
|
||||||
INDEXENTRYLENGTH = INDEXENTRYLENGTH
|
INDEXENTRYLENGTH = INDEXENTRYLENGTH
|
||||||
|
|
||||||
|
VERSION = 0
|
||||||
|
|
||||||
def __init__(self, ui, packpath):
|
def __init__(self, ui, packpath):
|
||||||
super(mutablehistorypack, self).__init__(ui, packpath)
|
super(mutablehistorypack, self).__init__(ui, packpath)
|
||||||
self.pastfiles = {}
|
self.pastfiles = {}
|
||||||
@ -379,7 +390,8 @@ class mutablehistorypack(basepack.mutablebasepack):
|
|||||||
files = sorted(files)
|
files = sorted(files)
|
||||||
|
|
||||||
rawindex = ""
|
rawindex = ""
|
||||||
|
fmt = self.INDEXFORMAT
|
||||||
for namehash, offset, size in files:
|
for namehash, offset, size in files:
|
||||||
rawindex += struct.pack(INDEXFORMAT, namehash, offset, size)
|
rawindex += struct.pack(fmt, namehash, offset, size)
|
||||||
|
|
||||||
return rawindex
|
return rawindex
|
||||||
|
Loading…
Reference in New Issue
Block a user