facebook/sapling
1
1
Fork 0
mirror of https://github.com/facebook/sapling.git synced 2024-10-07 07:17:55 +03:00
Code Issues Projects Releases Wiki Activity

util: increase filechunkiter size to 128k

Browse Source 
util.filechunkiter has been using a chunk size of 64k for more than 10 years,
also in years where Moore's law still was a law. It is probably ok to bump it
now and perhaps get a slight win in some cases.

Also, largefiles have been using 128k for a long time. Specifying that size
multiple times (or forgetting to do it) seems a bit stupid. Decreasing it to
64k also seems unfortunate.

Thus, we will set the default chunksize to 128k and use the default everywhere.
This commit is contained in:
Mads Kiilerich 2016-10-14 01:53:15 +02:00
parent f926541834
commit 39f2a13215
5 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hgext/largefiles/lfutil.py
View File

@ -372,7 +372,7 @@ def hashfile(file):
return '' return ''
hasher = hashlib.sha1('') hasher = hashlib.sha1('')
with open(file, 'rb') as fd: with open(file, 'rb') as fd:
for data in util.filechunkiter(fd, 128 * 1024): for data in util.filechunkiter(fd):
hasher.update(data) hasher.update(data)
return hasher.hexdigest() return hasher.hexdigest()

2
hgext/largefiles/overrides.py
View File

@ -1356,7 +1356,7 @@ def overridecat(orig, ui, repo, file1, *pats, **opts):
'downloaded') % lf) 'downloaded') % lf)
path = lfutil.usercachepath(repo.ui, hash) path = lfutil.usercachepath(repo.ui, hash)
with open(path, "rb") as fpin: with open(path, "rb") as fpin:
for chunk in util.filechunkiter(fpin, 128 * 1024): for chunk in util.filechunkiter(fpin):
fp.write(chunk) fp.write(chunk)
err = 0 err = 0
return err return err

2
hgext/largefiles/proto.py
View File

@ -134,7 +134,7 @@ def wirereposetup(ui, repo):
length)) length))
# SSH streams will block if reading more than length # SSH streams will block if reading more than length
for chunk in util.filechunkiter(stream, 128 * 1024, length): for chunk in util.filechunkiter(stream, limit=length):
yield chunk yield chunk
# HTTP streams must hit the end to process the last empty # HTTP streams must hit the end to process the last empty
# chunk of Chunked-Encoding so the connection can be reused. # chunk of Chunked-Encoding so the connection can be reused.

4
mercurial/util.py
View File

@ -1684,9 +1684,9 @@ class chunkbuffer(object):
return ''.join(buf) return ''.join(buf)
def filechunkiter(f, size=65536, limit=None): def filechunkiter(f, size=131072, limit=None):
"""Create a generator that produces the data in the file size """Create a generator that produces the data in the file size
(default 65536) bytes at a time, up to optional limit (default is (default 131072) bytes at a time, up to optional limit (default is
to read all data). Chunks may be less than size bytes if the to read all data). Chunks may be less than size bytes if the
chunk is the last chunk in the file, or the file is a socket or chunk is the last chunk in the file, or the file is a socket or
some other type of file that sometimes reads less data than is some other type of file that sometimes reads less data than is

2
tests/test-largefiles-small-disk.t
View File

@ -11,7 +11,7 @@ Test how largefiles abort in case the disk runs full
> shutil.copyfileobj = copyfileobj > shutil.copyfileobj = copyfileobj
> # > #
> # this makes the rewritten code abort: > # this makes the rewritten code abort:
> def filechunkiter(f, size=65536, limit=None): > def filechunkiter(f, size=131072, limit=None):
> yield f.read(4) > yield f.read(4)
> raise IOError(errno.ENOSPC, os.strerror(errno.ENOSPC)) > raise IOError(errno.ENOSPC, os.strerror(errno.ENOSPC))
> util.filechunkiter = filechunkiter > util.filechunkiter = filechunkiter