facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-07 07:17:55 +03:00
Code Issues Projects Releases Wiki Activity

packs: add entry count to pack index

Browse Source 
Summary:
Previously, we used the length of the index file to determine the upper bounds
of the bisect. In a future patch we'll want to add more data to the end of the
index file, so we need to record how long the index portion of the index is.
This patch adds that information.

Test Plan: Ran the tests.

Reviewers: #mercurial, quark

Reviewed By: quark

Subscribers: mjpieters

Differential Revision: https://phabricator.intern.facebook.com/D4983682

Signature: t1:4983682:1493693255:57ab9af2030847fedff05b6755113ba8ce0c933b
This commit is contained in:
Durham Goode 2017-05-03 10:19:45 -07:00
parent 8674a90bb5
commit d1a927d335
7 changed files with 37 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
cdatapack/cdatapack.c
View File

@ -248,9 +248,14 @@ datapack_handle_t *open_datapack(
handle->status = DATAPACK_HANDLE_OOM;
goto error_cleanup;
}
size_t index_offset = 0;
if (handle->version == 1) {
index_offset = 8;
}
handle->index_table = (disk_index_entry_t *)
(((const char *) handle->index_mmap) +
sizeof(disk_index_header_t) +
index_offset +
(sizeof(index_offset_t) * fanout_count));
disk_index_entry_t *index_end = (disk_index_entry_t *)
(((const char *) handle->index_mmap) + handle->index_file_sz);

25
remotefilelog/basepack.py
View File

@ -173,9 +173,9 @@ class basepack(versionmixin):
self._checkversion(version)
if 0b10000000 & config:
self.params = indexparams(LARGEFANOUTPREFIX)
self.params = indexparams(LARGEFANOUTPREFIX, version)
else:
self.params = indexparams(SMALLFANOUTPREFIX)
self.params = indexparams(SMALLFANOUTPREFIX, version)
@util.propertycache
def _fanouttable(self):
@ -188,6 +188,15 @@ class basepack(versionmixin):
fanouttable.append(fanoutentry)
return fanouttable
@util.propertycache
def _indexend(self):
if self.VERSION == 0:
return self.indexsize
else:
nodecount = struct.unpack_from('!Q', self._index,
self.params.indexstart - 8)[0]
return self.params.indexstart + nodecount * self.INDEXENTRYLENGTH
def freememory(self):
"""Unmap and remap the memory to free it up after known expensive
operations. Return True if self._data nad self._index were reloaded.
@ -305,9 +314,9 @@ class mutablebasepack(versionmixin):
largefanout = len(self.entries) > SMALLFANOUTCUTOFF
if largefanout:
params = indexparams(LARGEFANOUTPREFIX)
params = indexparams(LARGEFANOUTPREFIX, self.VERSION)
else:
params = indexparams(SMALLFANOUTPREFIX)
params = indexparams(SMALLFANOUTPREFIX, self.VERSION)
fanouttable = [EMPTYFANOUT] * params.fanoutcount
@ -332,10 +341,13 @@ class mutablebasepack(versionmixin):
last = offset
rawfanouttable += struct.pack('!I', offset)
rawentrieslength = struct.pack('!Q', len(self.entries))
rawindex = self.createindex(locations)
self._writeheader(params)
self.idxfp.write(rawfanouttable)
if self.VERSION == 1:
self.idxfp.write(rawentrieslength)
self.idxfp.write(rawindex)
self.idxfp.close()
@ -356,7 +368,7 @@ class indexparams(object):
__slots__ = ('fanoutprefix', 'fanoutstruct', 'fanoutcount', 'fanoutsize',
'indexstart')
def __init__(self, prefixsize):
def __init__(self, prefixsize, version):
self.fanoutprefix = prefixsize
# The struct pack format for fanout table location (i.e. the format that
@ -376,3 +388,6 @@ class indexparams(object):
self.fanoutsize = self.fanoutcount * 4
self.indexstart = FANOUTSTART + self.fanoutsize
if version == 1:
# Skip the index length
self.indexstart += 8

4
remotefilelog/datapack.py
View File

@ -184,6 +184,8 @@ class datapack(basepack.basepack):
fanout = self._fanouttable
start = fanout[fanoutkey] + params.indexstart
indexend = self._indexend
# Scan forward to find the first non-same entry, which is the upper
# bound.
for i in xrange(fanoutkey + 1, params.fanoutcount):
@ -191,7 +193,7 @@ class datapack(basepack.basepack):
if end != start:
break
else:
end = self.indexsize
end = indexend
# Bisect between start and end to find node
index = self._index

4
remotefilelog/historypack.py
View File

@ -185,12 +185,14 @@ class historypack(basepack.basepack):
fanout = self._fanouttable
start = fanout[fanoutkey] + params.indexstart
indexend = self._indexend
for i in xrange(fanoutkey + 1, params.fanoutcount):
end = fanout[i] + params.indexstart
if end != start:
break
else:
end = self.indexsize
end = indexend
# Bisect between start and end to find node
startnode = self._index[start:start + NODELENGTH]

4
tests/test-remotefilelog-repack-fast.t
View File

@ -45,9 +45,9 @@
# Test that the packs are readonly
$ ls_l $CACHEDIR/master/packs
-r--r--r-- 1062 276d308429d0303762befa376788300f0310f90e.histidx
-r--r--r-- 1070 276d308429d0303762befa376788300f0310f90e.histidx
-r--r--r-- 172 276d308429d0303762befa376788300f0310f90e.histpack
-r--r--r-- 1066 8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx
-r--r--r-- 1074 8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx
-r--r--r-- 69 8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack
# Test that the data in the new packs is accessible

4
tests/test-remotefilelog-repack.t
View File

@ -44,9 +44,9 @@
# Test that the packs are readonly
$ ls_l $CACHEDIR/master/packs
-r--r--r-- 1062 276d308429d0303762befa376788300f0310f90e.histidx
-r--r--r-- 1070 276d308429d0303762befa376788300f0310f90e.histidx
-r--r--r-- 172 276d308429d0303762befa376788300f0310f90e.histpack
-r--r--r-- 1066 8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx
-r--r--r-- 1074 8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.dataidx
-r--r--r-- 69 8e25dec685d5e0bb1f1b39df3acebda0e0d75c6e.datapack
# Test that the data in the new packs is accessible

4
tests/test-treemanifest.t
View File

@ -54,7 +54,7 @@ Test autocreatetrees
$ ls_l $CACHEDIR/master/packs/manifests
-r--r--r-- 1146 678f597a73b2b96f2e120c84ef8a84069a250266.dataidx
-r--r--r-- 315 678f597a73b2b96f2e120c84ef8a84069a250266.datapack
-r--r--r-- 1098 ed1a27864c5d25f144a51961ad6e79088f2a7571.histidx
-r--r--r-- 1106 ed1a27864c5d25f144a51961ad6e79088f2a7571.histidx
-r--r--r-- 265 ed1a27864c5d25f144a51961ad6e79088f2a7571.histpack
$ hg debugdatapack $CACHEDIR/master/packs/manifests/678f597a73b2b96f2e120c84ef8a84069a250266
@ -77,7 +77,7 @@ Test that commit creates local trees
$ echo z >> subdir/z
$ hg commit -qAm 'modify subdir/z'
$ ls_l .hg/store/packs/manifests
-r--r--r-- 1098 57710544ca24ac4f36682ec279959879c92a3275.histidx
-r--r--r-- 1106 57710544ca24ac4f36682ec279959879c92a3275.histidx
-r--r--r-- 183 57710544ca24ac4f36682ec279959879c92a3275.histpack
-r--r--r-- 1106 a7f7e084adff88a01cf76909345be1e56ee704a9.dataidx
-r--r--r-- 254 a7f7e084adff88a01cf76909345be1e56ee704a9.datapack